Skip to content

Commit

Permalink
YAML test framework: re-introduce requires section and `cluster_fea…
Browse files Browse the repository at this point in the history 
…tures` conditions (elastic#105763)
  • Loading branch information
@ldematte
ldematte authored Mar 5, 2024
1 parent e9ff896 commit 1e76b18
Show file tree
Hide file tree
Showing 4 changed files with 252 additions and 28 deletions.
23 changes: 12 additions & 11 deletions ...st-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/ClientYamlTestSuite.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ private static Stream<String> validateExecutableSections(
.filter(section -> false == section.getExpectedWarningHeaders().isEmpty())
.filter(section -> false == hasYamlRunnerFeature("warnings", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [warnings] section without a corresponding ["skip": "features": "warnings"] \
attempted to add a [do] with a [warnings] section without a corresponding ["requires": "test_runner_features": "warnings"] \
so runners that do not support the [warnings] section can skip the test at line [%d]\
""", section.getLocation().lineNumber()));

Expand All @@ -190,7 +190,7 @@ private static Stream<String> validateExecutableSections(
.filter(section -> false == hasYamlRunnerFeature("warnings_regex", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [warnings_regex] section without a corresponding \
["skip": "features": "warnings_regex"] so runners that do not support the [warnings_regex] \
["requires": "test_runner_features": "warnings_regex"] so runners that do not support the [warnings_regex] \
section can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
Expand All @@ -204,7 +204,7 @@ private static Stream<String> validateExecutableSections(
.filter(section -> false == hasYamlRunnerFeature("allowed_warnings", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [allowed_warnings] section without a corresponding \
["skip": "features": "allowed_warnings"] so runners that do not support the [allowed_warnings] \
["requires": "test_runner_features": "allowed_warnings"] so runners that do not support the [allowed_warnings] \
section can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
Expand All @@ -218,8 +218,8 @@ private static Stream<String> validateExecutableSections(
.filter(section -> false == hasYamlRunnerFeature("allowed_warnings_regex", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [allowed_warnings_regex] section without a corresponding \
["skip": "features": "allowed_warnings_regex"] so runners that do not support the [allowed_warnings_regex] \
section can skip the test at line [%d]\
["requires": "test_runner_features": "allowed_warnings_regex"] so runners that do not support the \
[allowed_warnings_regex] section can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);

Expand All @@ -232,7 +232,7 @@ private static Stream<String> validateExecutableSections(
.filter(section -> false == hasYamlRunnerFeature("node_selector", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [node_selector] section without a corresponding \
["skip": "features": "node_selector"] so runners that do not support the [node_selector] section \
["requires": "test_runner_features": "node_selector"] so runners that do not support the [node_selector] section \
can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
Expand All @@ -243,7 +243,7 @@ private static Stream<String> validateExecutableSections(
.filter(section -> section instanceof ContainsAssertion)
.filter(section -> false == hasYamlRunnerFeature("contains", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [contains] assertion without a corresponding ["skip": "features": "contains"] \
attempted to add a [contains] assertion without a corresponding ["requires": "test_runner_features": "contains"] \
so runners that do not support the [contains] assertion can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
Expand All @@ -256,8 +256,9 @@ private static Stream<String> validateExecutableSections(
.filter(section -> false == section.getApiCallSection().getHeaders().isEmpty())
.filter(section -> false == hasYamlRunnerFeature("headers", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [do] with a [headers] section without a corresponding ["skip": "features": "headers"] \
so runners that do not support the [headers] section can skip the test at line [%d]\
attempted to add a [do] with a [headers] section without a corresponding \
["requires": "test_runner_features": "headers"] so runners that do not support the [headers] section \
can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);

Expand All @@ -267,7 +268,7 @@ private static Stream<String> validateExecutableSections(
.filter(section -> section instanceof CloseToAssertion)
.filter(section -> false == hasYamlRunnerFeature("close_to", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add a [close_to] assertion without a corresponding ["skip": "features": "close_to"] \
attempted to add a [close_to] assertion without a corresponding ["requires": "test_runner_features": "close_to"] \
so runners that do not support the [close_to] assertion can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
Expand All @@ -278,7 +279,7 @@ private static Stream<String> validateExecutableSections(
.filter(section -> section instanceof IsAfterAssertion)
.filter(section -> false == hasYamlRunnerFeature("is_after", testSection, setupSection, teardownSection))
.map(section -> String.format(Locale.ROOT, """
attempted to add an [is_after] assertion without a corresponding ["skip": "features": "is_after"] \
attempted to add an [is_after] assertion without a corresponding ["requires": "test_runner_features": "is_after"] \
so runners that do not support the [is_after] assertion can skip the test at line [%d]\
""", section.getLocation().lineNumber()))
);
Expand Down
97 changes: 95 additions & 2 deletions ...st-runner/src/main/java/org/elasticsearch/test/rest/yaml/section/PrerequisiteSection.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@

import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.test.rest.yaml.ClientYamlTestExecutionContext;
import org.elasticsearch.test.rest.yaml.Features;
import org.elasticsearch.xcontent.XContentLocation;
Expand All @@ -17,7 +18,9 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;

/**
Expand All @@ -34,9 +37,13 @@ public class PrerequisiteSection {
static class PrerequisiteSectionBuilder {
String skipVersionRange = null;
String skipReason = null;
String requiresReason = null;
List<String> requiredYamlRunnerFeatures = new ArrayList<>();
List<String> skipOperatingSystems = new ArrayList<>();

Set<String> skipClusterFeatures = new HashSet<>();
Set<String> requiredClusterFeatures = new HashSet<>();

enum XPackRequired {
NOT_SPECIFIED,
YES,
Expand All @@ -56,6 +63,11 @@ public PrerequisiteSectionBuilder setSkipReason(String skipReason) {
return this;
}

public PrerequisiteSectionBuilder setRequiresReason(String requiresReason) {
this.requiresReason = requiresReason;
return this;
}

public PrerequisiteSectionBuilder requireYamlRunnerFeature(String featureName) {
requiredYamlRunnerFeatures.add(featureName);
return this;
Expand All @@ -79,6 +91,16 @@ public PrerequisiteSectionBuilder skipIfXPack() {
return this;
}

public PrerequisiteSectionBuilder skipIfClusterFeature(String featureName) {
skipClusterFeatures.add(featureName);
return this;
}

public PrerequisiteSectionBuilder requireClusterFeature(String featureName) {
requiredClusterFeatures.add(featureName);
return this;
}

public PrerequisiteSectionBuilder skipIfOs(String osName) {
this.skipOperatingSystems.add(osName);
return this;
Expand All @@ -88,7 +110,9 @@ void validate(XContentLocation contentLocation) {
if ((Strings.hasLength(skipVersionRange) == false)
&& requiredYamlRunnerFeatures.isEmpty()
&& skipOperatingSystems.isEmpty()
&& xpackRequired == XPackRequired.NOT_SPECIFIED) {
&& xpackRequired == XPackRequired.NOT_SPECIFIED
&& requiredClusterFeatures.isEmpty()
&& skipClusterFeatures.isEmpty()) {
throw new ParsingException(
contentLocation,
"at least one criteria (version, cluster features, runner features, os) is mandatory within a skip section"
Expand All @@ -100,13 +124,22 @@ void validate(XContentLocation contentLocation) {
if (skipOperatingSystems.isEmpty() == false && Strings.hasLength(skipReason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within skip os section");
}
if (skipClusterFeatures.isEmpty() == false && Strings.hasLength(skipReason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within skip cluster_features section");
}
if (requiredClusterFeatures.isEmpty() == false && Strings.hasLength(requiresReason) == false) {
throw new ParsingException(contentLocation, "reason is mandatory within requires cluster_features section");
}
// make feature "skip_os" mandatory if os is given, this is a temporary solution until language client tests know about os
if (skipOperatingSystems.isEmpty() == false && requiredYamlRunnerFeatures.contains("skip_os") == false) {
throw new ParsingException(contentLocation, "if os is specified, test runner feature [skip_os] must be set");
}
if (xpackRequired == XPackRequired.MISMATCHED) {
throw new ParsingException(contentLocation, "either [xpack] or [no_xpack] can be present, not both");
}
if (Sets.haveNonEmptyIntersection(skipClusterFeatures, requiredClusterFeatures)) {
throw new ParsingException(contentLocation, "a cluster feature can be specified either in [requires] or [skip], not both");
}
}

public PrerequisiteSection build() {
Expand All @@ -131,8 +164,14 @@ public PrerequisiteSection build() {
if (skipOperatingSystems.isEmpty() == false) {
skipCriteriaList.add(Prerequisites.skipOnOsList(skipOperatingSystems));
}
if (requiredClusterFeatures.isEmpty() == false) {
requiresCriteriaList.add(Prerequisites.requireClusterFeatures(requiredClusterFeatures));
}
if (skipClusterFeatures.isEmpty() == false) {
skipCriteriaList.add(Prerequisites.skipOnClusterFeatures(skipClusterFeatures));
}
}
return new PrerequisiteSection(skipCriteriaList, skipReason, requiresCriteriaList, null, requiredYamlRunnerFeatures);
return new PrerequisiteSection(skipCriteriaList, skipReason, requiresCriteriaList, requiresReason, requiredYamlRunnerFeatures);
}
}

Expand Down Expand Up @@ -160,6 +199,10 @@ static PrerequisiteSectionBuilder parseInternal(XContentParser parser) throws IO
parseSkipSection(parser, builder);
hasPrerequisiteSection = true;
maybeAdvanceToNextField(parser);
} else if ("requires".equals(parser.currentName())) {
parseRequiresSection(parser, builder);
hasPrerequisiteSection = true;
maybeAdvanceToNextField(parser);
} else {
unknownFieldName = true;
}
Expand Down Expand Up @@ -209,6 +252,8 @@ static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder b
parseFeatureField(parser.text(), builder);
} else if ("os".equals(currentFieldName)) {
builder.skipIfOs(parser.text());
} else if ("cluster_features".equals(currentFieldName)) {
builder.skipIfClusterFeature(parser.text());
} else {
throw new ParsingException(
parser.getTokenLocation(),
Expand All @@ -224,6 +269,54 @@ static void parseSkipSection(XContentParser parser, PrerequisiteSectionBuilder b
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.skipIfOs(parser.text());
}
} else if ("cluster_features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.skipIfClusterFeature(parser.text());
}
}
}
}
parser.nextToken();
}

static void parseRequiresSection(XContentParser parser, PrerequisiteSectionBuilder builder) throws IOException {
if (parser.nextToken() != XContentParser.Token.START_OBJECT) {
throw new IllegalArgumentException(
"Expected ["
+ XContentParser.Token.START_OBJECT
+ ", found ["
+ parser.currentToken()
+ "], the requires section is not properly indented"
);
}
String currentFieldName = null;
XContentParser.Token token;

while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if ("reason".equals(currentFieldName)) {
builder.setRequiresReason(parser.text());
} else if ("test_runner_features".equals(currentFieldName)) {
parseFeatureField(parser.text(), builder);
} else if ("cluster_features".equals(currentFieldName)) {
builder.requireClusterFeature(parser.text());
} else {
throw new ParsingException(
parser.getTokenLocation(),
"field " + currentFieldName + " not supported within requires section"
);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if ("test_runner_features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
parseFeatureField(parser.text(), builder);
}
} else if ("cluster_features".equals(currentFieldName)) {
while (parser.nextToken() != XContentParser.Token.END_ARRAY) {
builder.requireClusterFeature(parser.text());
}
}
}
}
Expand Down
Loading

0 comments on commit 1e76b18

Please sign in to comment.