-
Notifications
You must be signed in to change notification settings - Fork 33
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* Initial commit for cidr function Signed-off-by: Hendrik Saly <[email protected]> * Add inet.ipaddr library and implement basic cidr check Signed-off-by: Hendrik Saly <[email protected]> * Refactor for using lombok Signed-off-by: Hendrik Saly <[email protected]> * Add unittests Signed-off-by: Hendrik Saly <[email protected]> * add check for mixed ip address types like ipv4 and ipv6; add test with exception runtime exception in case of a mixed adress type Signed-off-by: Jens Schmidt <[email protected]> * Fix antlr Signed-off-by: Hendrik Saly <[email protected]> * rename cird function cirdmatch Signed-off-by: Jens Schmidt <[email protected]> * Fix imports Signed-off-by: Hendrik Saly <[email protected]> * Prepare integ tests Signed-off-by: Hendrik Saly <[email protected]> * Added IT tests Signed-off-by: Hendrik Saly <[email protected]> * Refactor SerializableUdf to be an interface Signed-off-by: Hendrik Saly <[email protected]> * Fix imports Signed-off-by: Hendrik Saly <[email protected]> * Added docs Signed-off-by: Hendrik Saly <[email protected]> --------- Signed-off-by: Hendrik Saly <[email protected]> Signed-off-by: Jens Schmidt <[email protected]> Co-authored-by: Jens Schmidt <[email protected]>
- Loading branch information
1 parent
d03cc8c
commit d2213c5
Showing
15 changed files
with
380 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| ## PPL IP Address Functions | ||
|
|
||
| ### `CIDRMATCH` | ||
|
|
||
| **Description** | ||
|
|
||
| `CIDRMATCH(ip, cidr)` checks if ip is within the specified cidr range. | ||
|
|
||
| **Argument type:** | ||
| - STRING, STRING | ||
| - Return type: **BOOLEAN** | ||
|
|
||
| Example: | ||
|
|
||
| os> source=ips | where cidrmatch(ip, '192.169.1.0/24') | fields ip | ||
| fetched rows / total rows = 1/1 | ||
| +--------------+ | ||
| | ip | | ||
| |--------------| | ||
| | 192.169.1.5 | | ||
| +--------------+ | ||
|
|
||
| os> source=ipsv6 | where cidrmatch(ip, '2003:db8::/32') | fields ip | ||
| fetched rows / total rows = 1/1 | ||
| +-----------------------------------------+ | ||
| | ip | | ||
| |-----------------------------------------| | ||
| | 2003:0db8:0000:0000:0000:0000:0000:0000 | | ||
| +-----------------------------------------+ | ||
|
|
||
| Note: | ||
| - `ip` can be an IPv4 or an IPv6 address | ||
| - `cidr` can be an IPv4 or an IPv6 block | ||
| - `ip` and `cidr` must be either both IPv4 or both IPv6 | ||
| - `ip` and `cidr` must both be valid and non-empty/non-null |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
96 changes: 96 additions & 0 deletions
96
.../src/integration/scala/org/opensearch/flint/spark/ppl/FlintSparkPPLCidrmatchITSuite.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,96 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.flint.spark.ppl | ||
|
|
||
| import org.apache.spark.SparkException | ||
| import org.apache.spark.sql.QueryTest | ||
| import org.apache.spark.sql.streaming.StreamTest | ||
|
|
||
| class FlintSparkPPLCidrmatchITSuite | ||
| extends QueryTest | ||
| with LogicalPlanTestUtils | ||
| with FlintPPLSuite | ||
| with StreamTest { | ||
|
|
||
| /** Test table and index name */ | ||
| private val testTable = "spark_catalog.default.flint_ppl_test" | ||
|
|
||
| override def beforeAll(): Unit = { | ||
| super.beforeAll() | ||
|
|
||
| // Create test table | ||
| createIpAddressTable(testTable) | ||
| } | ||
|
|
||
| protected override def afterEach(): Unit = { | ||
| super.afterEach() | ||
| // Stop all streaming jobs if any | ||
| spark.streams.active.foreach { job => | ||
| job.stop() | ||
| job.awaitTermination() | ||
| } | ||
| } | ||
|
|
||
| test("test cidrmatch for ipv4 for 192.168.1.0/24") { | ||
| val frame = sql(s""" | ||
| | source = $testTable | where isV6 = false and isValid = true and cidrmatch(ipAddress, '192.168.1.0/24') | ||
| | """.stripMargin) | ||
|
|
||
| val results = frame.collect() | ||
| assert(results.length == 2) | ||
| } | ||
|
|
||
| test("test cidrmatch for ipv4 for 192.169.1.0/24") { | ||
| val frame = sql(s""" | ||
| | source = $testTable | where isV6 = false and isValid = true and cidrmatch(ipAddress, '192.169.1.0/24') | ||
| | """.stripMargin) | ||
|
|
||
| val results = frame.collect() | ||
| assert(results.length == 0) | ||
| } | ||
|
|
||
| test("test cidrmatch for ipv6 for 2001:db8::/32") { | ||
| val frame = sql(s""" | ||
| | source = $testTable | where isV6 = true and isValid = true and cidrmatch(ipAddress, '2001:db8::/32') | ||
| | """.stripMargin) | ||
|
|
||
| val results = frame.collect() | ||
| assert(results.length == 3) | ||
| } | ||
|
|
||
| test("test cidrmatch for ipv6 for 2003:db8::/32") { | ||
| val frame = sql(s""" | ||
| | source = $testTable | where isV6 = true and isValid = true and cidrmatch(ipAddress, '2003:db8::/32') | ||
| | """.stripMargin) | ||
|
|
||
| val results = frame.collect() | ||
| assert(results.length == 0) | ||
| } | ||
|
|
||
| test("test cidrmatch for ipv6 with ipv4 cidr") { | ||
| val frame = sql(s""" | ||
| | source = $testTable | where isV6 = true and isValid = true and cidrmatch(ipAddress, '192.169.1.0/24') | ||
| | """.stripMargin) | ||
|
|
||
| assertThrows[SparkException](frame.collect()) | ||
| } | ||
|
|
||
| test("test cidrmatch for invalid ipv4 addresses") { | ||
| val frame = sql(s""" | ||
| | source = $testTable | where isV6 = false and isValid = false and cidrmatch(ipAddress, '192.169.1.0/24') | ||
| | """.stripMargin) | ||
|
|
||
| assertThrows[SparkException](frame.collect()) | ||
| } | ||
|
|
||
| test("test cidrmatch for invalid ipv6 addresses") { | ||
| val frame = sql(s""" | ||
| | source = $testTable | where isV6 = true and isValid = false and cidrmatch(ipAddress, '2003:db8::/32') | ||
| | """.stripMargin) | ||
|
|
||
| assertThrows[SparkException](frame.collect()) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
35 changes: 35 additions & 0 deletions
35
ppl-spark-integration/src/main/java/org/opensearch/sql/ast/expression/Cidr.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,35 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.ast.expression; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.EqualsAndHashCode; | ||
| import lombok.Getter; | ||
| import lombok.ToString; | ||
| import org.opensearch.sql.ast.AbstractNodeVisitor; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.List; | ||
|
|
||
| /** AST node that represents CIDR function. */ | ||
| @AllArgsConstructor | ||
| @Getter | ||
| @EqualsAndHashCode(callSuper = false) | ||
| @ToString | ||
| public class Cidr extends UnresolvedExpression { | ||
| private UnresolvedExpression ipAddress; | ||
| private UnresolvedExpression cidrBlock; | ||
|
|
||
| @Override | ||
| public List<UnresolvedExpression> getChild() { | ||
| return Arrays.asList(ipAddress, cidrBlock); | ||
| } | ||
|
|
||
| @Override | ||
| public <T, C> T accept(AbstractNodeVisitor<T, C> nodeVisitor, C context) { | ||
| return nodeVisitor.visitCidr(this, context); | ||
| } | ||
| } |
57 changes: 57 additions & 0 deletions
57
...ark-integration/src/main/java/org/opensearch/sql/expression/function/SerializableUdf.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.sql.expression.function; | ||
|
|
||
| import inet.ipaddr.AddressStringException; | ||
| import inet.ipaddr.IPAddressString; | ||
| import inet.ipaddr.IPAddressStringParameters; | ||
| import scala.Function2; | ||
| import scala.Serializable; | ||
| import scala.runtime.AbstractFunction2; | ||
|
|
||
|
|
||
| public interface SerializableUdf { | ||
|
|
||
| Function2<String,String,Boolean> cidrFunction = new SerializableAbstractFunction2<>() { | ||
|
|
||
| IPAddressStringParameters valOptions = new IPAddressStringParameters.Builder() | ||
| .allowEmpty(false) | ||
| .setEmptyAsLoopback(false) | ||
| .allow_inet_aton(false) | ||
| .allowSingleSegment(false) | ||
| .toParams(); | ||
|
|
||
| @Override | ||
| public Boolean apply(String ipAddress, String cidrBlock) { | ||
|
|
||
| IPAddressString parsedIpAddress = new IPAddressString(ipAddress, valOptions); | ||
|
|
||
| try { | ||
| parsedIpAddress.validate(); | ||
| } catch (AddressStringException e) { | ||
| throw new RuntimeException("The given ipAddress '"+ipAddress+"' is invalid. It must be a valid IPv4 or IPv6 address. Error details: "+e.getMessage()); | ||
| } | ||
|
|
||
| IPAddressString parsedCidrBlock = new IPAddressString(cidrBlock, valOptions); | ||
|
|
||
| try { | ||
| parsedCidrBlock.validate(); | ||
| } catch (AddressStringException e) { | ||
| throw new RuntimeException("The given cidrBlock '"+cidrBlock+"' is invalid. It must be a valid CIDR or netmask. Error details: "+e.getMessage()); | ||
| } | ||
|
|
||
| if(parsedIpAddress.isIPv4() && parsedCidrBlock.isIPv6() || parsedIpAddress.isIPv6() && parsedCidrBlock.isIPv4()) { | ||
| throw new RuntimeException("The given ipAddress '"+ipAddress+"' and cidrBlock '"+cidrBlock+"' are not compatible. Both must be either IPv4 or IPv6."); | ||
| } | ||
|
|
||
| return parsedCidrBlock.contains(parsedIpAddress); | ||
| } | ||
| }; | ||
|
|
||
| abstract class SerializableAbstractFunction2<T1,T2,R> extends AbstractFunction2<T1,T2,R> | ||
| implements Serializable { | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.