Merge pull request #2643 from Fork-World/2641-fail-tag

#2641 introduce @fail tag to mark tests which are expected to fail

Author: ptrthomas

README.md

@@ -4328,6 +4328,7 @@ For completeness, the "built-in" tags are the following:
 Tag | Description
 --- | -----------
 `@ignore` | Any `Scenario` with (or that has inherited) this tag will be skipped at run-time. This does not apply to anything that is "called" though
+`@fail` | Any `Scenario` with (or that has inherited) this tag will be expected to fail. This can be used if e.g. tests are written before fixes
 `@parallel` | See [`@parallel=false`](#parallelfalse)
 `@report` | See [`@report=false`](#reportfalse)
 `@setup` | See [`@setup`](#setup)

karate-core/src/main/java/com/intuit/karate/core/ScenarioResult.java

@@ -39,7 +39,7 @@
  */
 public class ScenarioResult implements Comparable<ScenarioResult> {
 
-    private final List<StepResult> stepResults = new ArrayList();
+    private final List<StepResult> stepResults = new ArrayList<>();
     private final Scenario scenario;
 
     private StepResult failedStep;
@@ -363,4 +363,7 @@ public String toString() {
         return failedStep == null ? scenario.toString() : failedStep + "";
     }
 
+    public void ignoreFailedStep() {
+        failedStep = null;
+    }
 }

karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java

@@ -46,6 +46,7 @@
  */
 public class ScenarioRuntime implements Runnable {
 
+    public static final String EXPECT_TEST_TO_FAIL_BECAUSE_OF_FAIL_TAG = "Expect test to fail because of @fail tag";
     public final Logger logger;
     public final FeatureRuntime featureRuntime;
     public final ScenarioCall caller;
@@ -510,6 +511,15 @@ public void afterRun() {
                 engine.stop(currentStepResult);
             }
             addStepLogEmbedsAndCallResults();
+            if (tags.contains(Tag.FAIL)) {
+                if (result.isFailed()) {
+                    result.ignoreFailedStep();
+                    result.addFakeStepResult(EXPECT_TEST_TO_FAIL_BECAUSE_OF_FAIL_TAG, null);
+                } else {
+                    result.addFakeStepResult(EXPECT_TEST_TO_FAIL_BECAUSE_OF_FAIL_TAG,
+                            new Throwable(EXPECT_TEST_TO_FAIL_BECAUSE_OF_FAIL_TAG));
+                }
+            }
         } catch (Exception e) {
             error = e;
             logError("scenario [cleanup] failed\n" + e.getMessage());

karate-core/src/main/java/com/intuit/karate/core/Tag.java

@@ -37,6 +37,7 @@ public class Tag {
     public static final String ENV = "env";
     public static final String ENVNOT = "envnot";
     public static final String SETUP = "setup";
+    public static final String FAIL = "fail";
 
     private final int line;
     private final String text;

karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java

@@ -60,6 +60,18 @@ private void matchContains(Object actual, Object expected) {
         assertTrue(mr.pass, mr.message);
     }
 
+    @Test
+    void testFailTag() {
+        fail = false;
+        run("fail-tag.feature");
+    }
+
+    @Test
+    void testFailTagFailure() {
+        fail = true;
+        run("fail-tag-failure.feature");
+    }
+
     @Test
     void testFail1() {
         fail = true;

karate-core/src/test/java/com/intuit/karate/core/fail-tag-failure.feature

@@ -0,0 +1,6 @@
+Feature: fail tag failure
+
+@fail
+Scenario:
+* def a = 1 + 2
+* match a == 3

karate-core/src/test/java/com/intuit/karate/core/fail-tag.feature

@@ -0,0 +1,6 @@
+Feature: fail tag
+
+@fail
+Scenario:
+* def a = 1 + 2
+* match a == 4