diff --git a/docs/routing-rules.md b/docs/routing-rules.md index 78def563c..d10af333f 100644 --- a/docs/routing-rules.md +++ b/docs/routing-rules.md @@ -296,8 +296,8 @@ actions: ``` This can difficult to maintain with more rules. To have better control over the -execution of rules, we can use rule priorities and composite rules. Overall, -priorities, composite rules, and other constructs that MVEL support allows +execution of rules, we can use rule priorities. Overall, +priorities and other constructs that MVEL support allows you to express your routing logic. #### Rule priority @@ -328,99 +328,12 @@ that the first rule (priority 0) is fired before the second rule (priority 1). Thus `routingGroup` is set to `etl` and then to `etl-special`, so the `routingGroup` is always `etl-special` in the end. -More specific rules must be set to a lesser priority so they are evaluated last -to set a `routingGroup`. To further control the execution of rules, for example -to have only one rule fire, you can use composite rules. - -##### Composite rules - -First, please refer to the [easy-rule composite rules documentation](https://github.com/j-easy/easy-rules/wiki/defining-rules#composite-rules). - -The preceding section covers how to control the order of rule execution using -priorities. In addition, you can configure evaluation so that only the first -rule matched fires (the highest priority one) and the rest is ignored. You can -use `ActivationRuleGroup` to achieve this: - -```yaml ---- -name: "airflow rule group" -description: "routing rules for query from airflow" -compositeRuleType: "ActivationRuleGroup" -composingRules: - - name: "airflow special" - description: "if query from airflow with special label, route to etl-special group" - priority: 0 - condition: 'request.getHeader("X-Trino-Source") == "airflow" && request.getHeader("X-Trino-Client-Tags") contains "label=special"' - actions: - - 'result.put("routingGroup", "etl-special")' - - name: "airflow" - description: "if query from airflow, route to etl group" - priority: 1 - condition: 'request.getHeader("X-Trino-Source") == "airflow"' - actions: - - 'result.put("routingGroup", "etl")' -``` - -Note that the priorities have switched. The more specific rule has a higher -priority, since it should fire first. A query coming from airflow with special -label is matched to the "airflow special" rule first, since it's higher -priority, and the second rule is ignored. A query coming from airflow with no -labels does not match the first rule, and is then tested and matched to the -second rule. - -You can also use `ConditionalRuleGroup` and `ActivationRuleGroup` to implement -an if/else workflow. The following logic in pseudocode: - -```text -if source == "airflow": - if clientTags["label"] == "foo": - return "etl-foo" - else if clientTags["label"] = "bar": - return "etl-bar" - else - return "etl" -``` - -This logic can be implemented with the following rules: - -```yaml -name: "airflow rule group" -description: "routing rules for query from airflow" -compositeRuleType: "ConditionalRuleGroup" -composingRules: - - name: "main condition" - description: "source is airflow" - priority: 0 # rule with the highest priority acts as main condition - condition: 'request.getHeader("X-Trino-Source") == "airflow"' - actions: - - "" - - name: "airflow subrules" - compositeRuleType: "ActivationRuleGroup" # use ActivationRuleGroup to simulate if/else - composingRules: - - name: "label foo" - description: "label client tag is foo" - priority: 0 - condition: 'request.getHeader("X-Trino-Client-Tags") contains "label=foo"' - actions: - - 'result.put("routingGroup", "etl-foo")' - - name: "label bar" - description: "label client tag is bar" - priority: 0 - condition: 'request.getHeader("X-Trino-Client-Tags") contains "label=bar"' - actions: - - 'result.put("routingGroup", "etl-bar")' - - name: "airflow default" - description: "airflow queries default to etl" - condition: "true" - actions: - - 'result.put("routingGroup", "etl")' -``` +More specific rules must be set to a higher priority so they are evaluated last +to set a `routingGroup`. ##### If statements (MVEL Flow Control) -In the preceding section you see how `ConditionalRuleGroup` and -`ActivationRuleGroup` are used to implement an `if/else` workflow. You can -use MVEL support for `if` statements and other flow control. The following logic +You can use MVEL support for `if` statements and other flow control. The following logic in pseudocode: ```text diff --git a/gateway-ha/pom.xml b/gateway-ha/pom.xml index d8b56dc29..15b5af463 100644 --- a/gateway-ha/pom.xml +++ b/gateway-ha/pom.xml @@ -21,7 +21,6 @@ https://registry.npmmirror.com - 4.1.0 5.14.2 4.12.0 462 @@ -253,21 +252,9 @@ - org.jeasy - easy-rules-core - ${dep.jeasy.version} - - - - org.jeasy - easy-rules-mvel - ${dep.jeasy.version} - - - - org.jeasy - easy-rules-support - ${dep.jeasy.version} + org.mvel + mvel2 + 2.5.2.Final @@ -290,13 +277,6 @@ runtime - - org.mvel - mvel2 - 2.5.2.Final - runtime - - org.postgresql postgresql diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/MVELFileRoutingGroupSelector.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/MVELFileRoutingGroupSelector.java new file mode 100644 index 000000000..40d82c7c1 --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/router/MVELFileRoutingGroupSelector.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.gateway.ha.router; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLParser; +import io.trino.gateway.ha.config.RequestAnalyzerConfig; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class MVELFileRoutingGroupSelector + extends RulesRoutingGroupSelector +{ + Path rulesPath; + + MVELFileRoutingGroupSelector(String rulesPath, RequestAnalyzerConfig requestAnalyzerConfig) + { + super(requestAnalyzerConfig); + this.rulesPath = Paths.get(rulesPath); + + setRules(readRulesFromPath(this.rulesPath)); + } + + @Override + void reloadRules(long lastUpdatedTimeMillis) + { + try { + BasicFileAttributes attr = Files.readAttributes(this.rulesPath, BasicFileAttributes.class); + if (attr.lastModifiedTime().toMillis() > lastUpdatedTimeMillis) { + synchronized (this) { + if (attr.lastModifiedTime().toMillis() > lastUpdatedTimeMillis) { + List ruleList = readRulesFromPath(this.rulesPath); + setRules(ruleList); + } + } + } + } + catch (IOException e) { + throw new RuntimeException("Could not access rules file", e); + } + } + + public List readRulesFromPath(Path rulesPath) + { + ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory()); + try { + String content = Files.readString(rulesPath, UTF_8); + YAMLParser parser = new YAMLFactory().createParser(content); + List routingRulesList = new ArrayList<>(); + while (parser.nextToken() != null) { + MVELRoutingRule routingRules = yamlReader.readValue(parser, MVELRoutingRule.class); + routingRulesList.add(routingRules); + } + return routingRulesList; + } + catch (IOException e) { + throw new RuntimeException("Failed to read or parse routing rules configuration from path : " + rulesPath, e); + } + } +} diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/MVELRoutingRule.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/MVELRoutingRule.java new file mode 100644 index 000000000..11a9e8265 --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/router/MVELRoutingRule.java @@ -0,0 +1,60 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.gateway.ha.router; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; + +import static com.google.common.collect.ImmutableList.toImmutableList; +import static org.mvel2.MVEL.compileExpression; +import static org.mvel2.MVEL.executeExpression; + +public class MVELRoutingRule + extends RoutingRule +{ + @JsonCreator + public MVELRoutingRule( + @JsonProperty("name") String name, + @JsonProperty("description") String description, + @JsonProperty("priority") Integer priority, + @JsonProperty("condition") Serializable condition, + @JsonProperty("actions") List actions) + { + super( + name, + description, + priority, + condition instanceof String stringCondition ? compileExpression(stringCondition) : condition, + actions.stream().map(action -> { + if (action instanceof String stringAction) { + return compileExpression(stringAction); + } + else { + return action; + } + }).collect(toImmutableList())); + } + + @Override + public void evaluate(Map variables) + { + if ((boolean) executeExpression(condition, variables)) { + actions.forEach(action -> executeExpression(action, variables)); + } + } +} diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/RoutingGroupSelector.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/RoutingGroupSelector.java index ae6285e14..7bfe7ee37 100644 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/router/RoutingGroupSelector.java +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/router/RoutingGroupSelector.java @@ -39,7 +39,7 @@ static RoutingGroupSelector byRoutingGroupHeader() */ static RoutingGroupSelector byRoutingRulesEngine(String rulesConfigPath, RequestAnalyzerConfig requestAnalyzerConfig) { - return new RuleReloadingRoutingGroupSelector(rulesConfigPath, requestAnalyzerConfig); + return new MVELFileRoutingGroupSelector(rulesConfigPath, requestAnalyzerConfig); } /** diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/RoutingRule.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/RoutingRule.java new file mode 100644 index 000000000..b227fec73 --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/router/RoutingRule.java @@ -0,0 +1,59 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.gateway.ha.router; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.List; +import java.util.Map; + +import static java.util.Objects.requireNonNull; +import static java.util.Objects.requireNonNullElse; + +public abstract class RoutingRule + implements Comparable> +{ + String name; + String description; + Integer priority; + T condition; + List actions; + + @JsonCreator + public RoutingRule( + @JsonProperty("name") String name, + @JsonProperty("description") String description, + @JsonProperty("priority") Integer priority, + @JsonProperty("condition") T condition, + @JsonProperty("actions") List actions) + { + this.name = requireNonNull(name, "name is null"); + this.description = requireNonNullElse(description, ""); + this.priority = requireNonNullElse(priority, 0); + this.condition = requireNonNull(condition, "condition is null"); + this.actions = requireNonNull(actions, "actions is null"); + } + + public abstract void evaluate(Map variables); + + @Override + public int compareTo(RoutingRule o) + { + if (o == null) { + return 1; + } + return priority.compareTo(o.priority); + } +} diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/RuleReloadingRoutingGroupSelector.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/RuleReloadingRoutingGroupSelector.java deleted file mode 100644 index 4be89f39c..000000000 --- a/gateway-ha/src/main/java/io/trino/gateway/ha/router/RuleReloadingRoutingGroupSelector.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.trino.gateway.ha.router; - -import io.airlift.log.Logger; -import io.trino.gateway.ha.config.RequestAnalyzerConfig; -import jakarta.servlet.http.HttpServletRequest; -import org.jeasy.rules.api.Facts; -import org.jeasy.rules.api.Rules; -import org.jeasy.rules.api.RulesEngine; -import org.jeasy.rules.core.DefaultRulesEngine; -import org.jeasy.rules.mvel.MVELRuleFactory; -import org.jeasy.rules.support.reader.YamlRuleDefinitionReader; - -import java.io.FileReader; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.HashMap; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static java.nio.charset.StandardCharsets.UTF_8; - -public class RuleReloadingRoutingGroupSelector - implements RoutingGroupSelector -{ - private static final Logger log = Logger.get(RuleReloadingRoutingGroupSelector.class); - private final RulesEngine rulesEngine = new DefaultRulesEngine(); - private final MVELRuleFactory ruleFactory = new MVELRuleFactory(new YamlRuleDefinitionReader()); - private final String rulesConfigPath; - private volatile Rules rules = new Rules(); - private volatile long lastUpdatedTime; - private final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(true); - private final RequestAnalyzerConfig requestAnalyzerConfig; - private final TrinoRequestUser.TrinoRequestUserProvider trinoRequestUserProvider; - - RuleReloadingRoutingGroupSelector(String rulesConfigPath, RequestAnalyzerConfig requestAnalyzerConfig) - { - this.rulesConfigPath = rulesConfigPath; - this.requestAnalyzerConfig = requestAnalyzerConfig; - trinoRequestUserProvider = new TrinoRequestUser.TrinoRequestUserProvider(requestAnalyzerConfig); - try { - rules = ruleFactory.createRules( - new FileReader(rulesConfigPath, UTF_8)); - BasicFileAttributes attr = Files.readAttributes(Path.of(rulesConfigPath), - BasicFileAttributes.class); - lastUpdatedTime = attr.lastModifiedTime().toMillis(); - } - catch (Exception e) { - throw new RuntimeException("Error opening rules configuration file at " - + rulesConfigPath + "\n" - + "Using routing group header as default.", e); - } - } - - @Override - public String findRoutingGroup(HttpServletRequest request) - { - try { - BasicFileAttributes attr = Files.readAttributes(Path.of(rulesConfigPath), - BasicFileAttributes.class); - log.debug("File modified time: %s. lastUpdatedTime: %s", attr.lastModifiedTime(), lastUpdatedTime); - if (attr.lastModifiedTime().toMillis() > lastUpdatedTime) { - Lock writeLock = readWriteLock.writeLock(); - writeLock.lock(); - try { - if (attr.lastModifiedTime().toMillis() > lastUpdatedTime) { - // This check is performed again to prevent parsing the rules twice in case another - // thread finds the condition true and acquires the lock before this one - log.info("Updating rules to file modified at %s", attr.lastModifiedTime()); - rules = ruleFactory.createRules( - new FileReader(rulesConfigPath, UTF_8)); - lastUpdatedTime = attr.lastModifiedTime().toMillis(); - } - } - finally { - writeLock.unlock(); - } - } - - Facts facts = new Facts(); - HashMap result = new HashMap(); - - facts.put("request", request); - if (requestAnalyzerConfig.isAnalyzeRequest()) { - TrinoQueryProperties trinoQueryProperties = new TrinoQueryProperties( - request, - requestAnalyzerConfig.isClientsUseV2Format(), - requestAnalyzerConfig.getMaxBodySize()); - TrinoRequestUser trinoRequestUser = trinoRequestUserProvider.getInstance(request); - facts.put("trinoQueryProperties", trinoQueryProperties); - facts.put("trinoRequestUser", trinoRequestUser); - } - facts.put("result", result); - Lock readLock = readWriteLock.readLock(); - readLock.lock(); - try { - rulesEngine.fire(rules, facts); - } - finally { - readLock.unlock(); - } - return result.get("routingGroup"); - } - catch (Exception e) { - log.error(e, "Error opening rules configuration file, using " - + "routing group header as default."); - // Invalid rules could lead to perf problems as every thread goes into the writeLock - // block until the issue is resolved - } - return request.getHeader(ROUTING_GROUP_HEADER); - } -} diff --git a/gateway-ha/src/main/java/io/trino/gateway/ha/router/RulesRoutingGroupSelector.java b/gateway-ha/src/main/java/io/trino/gateway/ha/router/RulesRoutingGroupSelector.java new file mode 100644 index 000000000..0be9280d0 --- /dev/null +++ b/gateway-ha/src/main/java/io/trino/gateway/ha/router/RulesRoutingGroupSelector.java @@ -0,0 +1,85 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.gateway.ha.router; + +import com.google.common.collect.ImmutableMap; +import io.trino.gateway.ha.config.RequestAnalyzerConfig; +import jakarta.servlet.http.HttpServletRequest; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.sort; + +public abstract class RulesRoutingGroupSelector> + implements RoutingGroupSelector +{ + public static final String RESULTS_ROUTING_GROUP_KEY = "routingGroup"; + + private List rules; + final boolean analyzeRequest; + final boolean clientsUseV2Format; + final int maxBodySize; + final TrinoRequestUser.TrinoRequestUserProvider trinoRequestUserProvider; + private volatile long lastUpdatedTimeMillis; + + RulesRoutingGroupSelector(RequestAnalyzerConfig requestAnalyzerConfig) + { + this(new ArrayList<>(), requestAnalyzerConfig); + } + + public RulesRoutingGroupSelector(List rules, RequestAnalyzerConfig requestAnalyzerConfig) + { + setRules(rules); + analyzeRequest = requestAnalyzerConfig.isAnalyzeRequest(); + clientsUseV2Format = requestAnalyzerConfig.isClientsUseV2Format(); + maxBodySize = requestAnalyzerConfig.getMaxBodySize(); + trinoRequestUserProvider = new TrinoRequestUser.TrinoRequestUserProvider(requestAnalyzerConfig); + } + + abstract void reloadRules(long lastUpdatedTimeMillis); + + void setRules(List rules) + { + this.rules = new ArrayList<>(rules); + lastUpdatedTimeMillis = System.currentTimeMillis(); + sort(this.rules); + } + + // TODO: add CRUD operations for the rules + + @Override + public String findRoutingGroup(HttpServletRequest request) + { + reloadRules(lastUpdatedTimeMillis); + Map result = new HashMap<>(); + Map variables; + if (analyzeRequest) { + TrinoQueryProperties trinoQueryProperties = new TrinoQueryProperties( + request, + clientsUseV2Format, + maxBodySize); + TrinoRequestUser trinoRequestUser = trinoRequestUserProvider.getInstance(request); + variables = ImmutableMap.of("result", result, "request", request, "trinoQueryProperties", trinoQueryProperties, "trinoRequestUser", trinoRequestUser); + } + else { + variables = ImmutableMap.of("result", result, "request", request); + } + + rules.forEach(rule -> rule.evaluate(variables)); + return result.get(RESULTS_ROUTING_GROUP_KEY); + } +} diff --git a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestRoutingGroupSelector.java b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestRoutingGroupSelector.java index 1066d19a7..abb7fe0d9 100644 --- a/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestRoutingGroupSelector.java +++ b/gateway-ha/src/test/java/io/trino/gateway/ha/router/TestRoutingGroupSelector.java @@ -67,7 +67,6 @@ static Stream provideRoutingRuleConfigFiles() String rulesDir = "src/test/resources/rules/"; return Stream.of( rulesDir + "routing_rules_atomic.yml", - rulesDir + "routing_rules_composite.yml", rulesDir + "routing_rules_priorities.yml", rulesDir + "routing_rules_if_statements.yml"); } diff --git a/gateway-ha/src/test/resources/rules/routing_rules_composite.yml b/gateway-ha/src/test/resources/rules/routing_rules_composite.yml deleted file mode 100644 index 2ddee5d9c..000000000 --- a/gateway-ha/src/test/resources/rules/routing_rules_composite.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: "airflow rule group" -description: "routing rules for query from airflow" -compositeRuleType: "ConditionalRuleGroup" -composingRules: - - name: "main condition" - description: "source is airflow" - priority: 0 - condition: "request.getHeader(\"X-Trino-Source\") == \"airflow\"" - actions: - - "" - - name: "airflow subrules" - compositeRuleType: "ActivationRuleGroup" - composingRules: - - name: "airflow special" - description: "special label" - priority: 0 - condition: "request.getHeader(\"X-Trino-Client-Tags\") contains \"label=special\"" - actions: - - "result.put(\"routingGroup\", \"etl-special\")" - - name: "airflow default" - description: "airflow queries default to etl" - condition: "true" - actions: - - "result.put(\"routingGroup\", \"etl\")"