Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#3145 Add IP Address Data Type #3175

Open
wants to merge 20 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion DEVELOPER_GUIDE.rst
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,7 @@ Sample test class:
Doctest
>>>>>>>

Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./dev/Doctest.md>`_.
Python doctest library makes our document executable which keeps it up-to-date to source code. The doc generator aforementioned served as scaffolding and generated many docs in short time. Now the examples inside is changed to doctest gradually. For more details please read `Doctest <./docs/dev/testing-doctest.md>`_.


Backports
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,7 @@ public static List<Argument> sortOptions() {
}

public static List<Argument> defaultSortFieldArgs() {
return exprList(argument("asc", booleanLiteral(true)), argument("type", nullLiteral()));
return exprList(argument("asc", booleanLiteral(true)));
}

public static Span span(UnresolvedExpression field, UnresolvedExpression value, SpanUnit unit) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_DOUBLE;
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_FLOAT;
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_INT;
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_IP;
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_LONG;
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_SHORT;
import static org.opensearch.sql.expression.function.BuiltinFunctionName.CAST_TO_STRING;
Expand Down Expand Up @@ -54,6 +55,7 @@ public class Cast extends UnresolvedExpression {
.put("time", CAST_TO_TIME.getName())
.put("timestamp", CAST_TO_TIMESTAMP.getName())
.put("datetime", CAST_TO_DATETIME.getName())
.put("ip", CAST_TO_IP.getName())
.build();

/** The source expression cast from. */
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*/

package org.opensearch.sql.data.model;

import inet.ipaddr.IPAddress;
import org.opensearch.sql.data.type.ExprCoreType;
import org.opensearch.sql.data.type.ExprType;
import org.opensearch.sql.utils.IPUtils;

/** Expression IP Address Value. */
public class ExprIpValue extends AbstractExprValue {
private final IPAddress value;

public ExprIpValue(String s) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit:

Suggested change
public ExprIpValue(String s) {
public ExprIpValue(String addressStr) {

value = IPUtils.toAddress(s);
}

@Override
public String value() {
return value.toCanonicalString();
}

@Override
public ExprType type() {
return ExprCoreType.IP;
}

@Override
public int compare(ExprValue other) {
return IPUtils.compare(value, ((ExprIpValue) other).value);
}

@Override
public boolean equal(ExprValue other) {
return compare(other) == 0;
}

@Override
public String toString() {
return String.format("IP %s", value());
}

@Override
public IPAddress ipValue() {
return value;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.sql.data.model;

import inet.ipaddr.IPAddress;
import java.io.Serializable;
import java.time.Instant;
import java.time.LocalDate;
Expand Down Expand Up @@ -102,6 +103,11 @@ default Double doubleValue() {
"invalid to get doubleValue from value of type " + type());
}

/** Get IP address value. */
default IPAddress ipValue() {
throw new ExpressionEvaluationException("invalid to get ipValue from value of type " + type());
}

/** Get string value. */
default String stringValue() {
throw new ExpressionEvaluationException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

package org.opensearch.sql.data.model;

import inet.ipaddr.IPAddress;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
Expand Down Expand Up @@ -75,6 +76,10 @@ public static ExprValue timestampValue(Instant value) {
return new ExprTimestampValue(value);
}

public static ExprValue ipValue(String value) {
return new ExprIpValue(value);
}

/** {@link ExprTupleValue} constructor. */
public static ExprValue tupleValue(Map<String, Object> map) {
LinkedHashMap<String, ExprValue> valueMap = new LinkedHashMap<>();
Expand Down Expand Up @@ -188,6 +193,10 @@ public static Map<String, ExprValue> getTupleValue(ExprValue exprValue) {
return exprValue.tupleValue();
}

public static IPAddress getIpValue(ExprValue exprValue) {
return exprValue.ipValue();
}

public static Boolean getBooleanValue(ExprValue exprValue) {
return exprValue.booleanValue();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ public enum ExprCoreType implements ExprType {
TIMESTAMP(STRING, DATE, TIME),
INTERVAL(UNDEFINED),

/** IP Address. */
IP(STRING),

/** Struct. */
STRUCT(UNDEFINED),

Expand Down
4 changes: 4 additions & 0 deletions core/src/main/java/org/opensearch/sql/expression/DSL.java
Original file line number Diff line number Diff line change
Expand Up @@ -835,6 +835,10 @@ public static FunctionExpression castTimestamp(Expression value) {
return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_TIMESTAMP, value);
}

public static FunctionExpression castIp(Expression value) {
return compile(FunctionProperties.None, BuiltinFunctionName.CAST_TO_IP, value);
}

public static FunctionExpression typeof(Expression value) {
return compile(FunctionProperties.None, BuiltinFunctionName.TYPEOF, value);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,6 +231,7 @@ public enum BuiltinFunctionName {
CAST_TO_TIME(FunctionName.of("cast_to_time")),
CAST_TO_TIMESTAMP(FunctionName.of("cast_to_timestamp")),
CAST_TO_DATETIME(FunctionName.of("cast_to_datetime")),
CAST_TO_IP(FunctionName.of("cast_to_ip")),
TYPEOF(FunctionName.of("typeof")),

/** Relevance Function. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,21 @@
package org.opensearch.sql.expression.ip;

import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN;
import static org.opensearch.sql.data.type.ExprCoreType.IP;
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
import static org.opensearch.sql.expression.function.FunctionDSL.define;
import static org.opensearch.sql.expression.function.FunctionDSL.impl;
import static org.opensearch.sql.expression.function.FunctionDSL.nullMissingHandling;

import inet.ipaddr.AddressStringException;
import inet.ipaddr.IPAddressString;
import inet.ipaddr.IPAddressStringParameters;
import inet.ipaddr.IPAddress;
import lombok.experimental.UtilityClass;
import org.opensearch.sql.data.model.ExprValue;
import org.opensearch.sql.data.model.ExprValueUtils;
import org.opensearch.sql.exception.SemanticCheckException;
import org.opensearch.sql.expression.function.BuiltinFunctionName;
import org.opensearch.sql.expression.function.BuiltinFunctionRepository;
import org.opensearch.sql.expression.function.DefaultFunctionResolver;
import org.opensearch.sql.utils.IPUtils;

/** Utility class that defines and registers IP functions. */
@UtilityClass
Expand All @@ -31,75 +31,35 @@ public void register(BuiltinFunctionRepository repository) {
}

private DefaultFunctionResolver cidrmatch() {

// TODO #3145: Add support for IP address data type.
return define(
BuiltinFunctionName.CIDRMATCH.getName(),
impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, STRING, STRING));
impl(nullMissingHandling(IPFunctions::exprCidrMatch), BOOLEAN, IP, STRING));
}

/**
* Returns whether the given IP address is within the specified inclusive CIDR IP address range.
* Supports both IPv4 and IPv6 addresses.
*
* @param addressExprValue IP address as a string (e.g. "198.51.100.14" or
* "2001:0db8::ff00:42:8329").
* @param rangeExprValue IP address range in CIDR notation as a string (e.g. "198.51.100.0/24" or
* @param addressExprValue IP address (e.g. "198.51.100.14" or "2001:0db8::ff00:42:8329").
* @param rangeExprValue IP address range string in CIDR notation (e.g. "198.51.100.0/24" or
* "2001:0db8::/32")
* @return true if the address is in the range; otherwise false.
* @throws SemanticCheckException if the address or range is not valid, or if they do not use the
* same version (IPv4 or IPv6).
*/
private ExprValue exprCidrMatch(ExprValue addressExprValue, ExprValue rangeExprValue) {

// TODO #3145: Update to support IP address data type.
String addressString = addressExprValue.stringValue();
String rangeString = rangeExprValue.stringValue();

final IPAddressStringParameters validationOptions =
new IPAddressStringParameters.Builder()
.allowEmpty(false)
.setEmptyAsLoopback(false)
.allow_inet_aton(false)
.allowSingleSegment(false)
.toParams();

// Get and validate IP address.
IPAddressString address =
new IPAddressString(addressExprValue.stringValue(), validationOptions);

try {
address.validate();
} catch (AddressStringException e) {
String msg =
String.format(
"IP address '%s' is not valid. Error details: %s", addressString, e.getMessage());
throw new SemanticCheckException(msg, e);
}

// Get and validate CIDR IP address range.
IPAddressString range = new IPAddressString(rangeExprValue.stringValue(), validationOptions);
IPAddress address = addressExprValue.ipValue();
IPAddress range = IPUtils.toRange(rangeExprValue.stringValue());

try {
range.validate();
} catch (AddressStringException e) {
String msg =
String.format(
"CIDR IP address range '%s' is not valid. Error details: %s",
rangeString, e.getMessage());
throw new SemanticCheckException(msg, e);
if (IPUtils.compare(address, range.getLower()) < 0) {
return ExprValueUtils.LITERAL_FALSE;
}

// Address and range must use the same IP version (IPv4 or IPv6).
if (address.isIPv4() ^ range.isIPv4()) {
String msg =
String.format(
"IP address '%s' and CIDR IP address range '%s' are not compatible. Both must be"
+ " either IPv4 or IPv6.",
addressString, rangeString);
throw new SemanticCheckException(msg);
if (IPUtils.compare(address, range.getUpper()) > 0) {
return ExprValueUtils.LITERAL_FALSE;
}

return ExprValueUtils.booleanValue(range.contains(address));
return ExprValueUtils.LITERAL_TRUE;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit:

return (IPUtils.compare(address, range.getLower()) < 0) || (IPUtils.compare(address, range.getUpper()) > 0) ? ExprValueUtils.LITERAL_FALSE : ExprValueUtils.LITERAL_TRUE;

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE;
import static org.opensearch.sql.data.type.ExprCoreType.FLOAT;
import static org.opensearch.sql.data.type.ExprCoreType.INTEGER;
import static org.opensearch.sql.data.type.ExprCoreType.IP;
import static org.opensearch.sql.data.type.ExprCoreType.LONG;
import static org.opensearch.sql.data.type.ExprCoreType.SHORT;
import static org.opensearch.sql.data.type.ExprCoreType.STRING;
Expand All @@ -31,6 +32,7 @@
import org.opensearch.sql.data.model.ExprDoubleValue;
import org.opensearch.sql.data.model.ExprFloatValue;
import org.opensearch.sql.data.model.ExprIntegerValue;
import org.opensearch.sql.data.model.ExprIpValue;
import org.opensearch.sql.data.model.ExprLongValue;
import org.opensearch.sql.data.model.ExprShortValue;
import org.opensearch.sql.data.model.ExprStringValue;
Expand All @@ -54,6 +56,7 @@ public static void register(BuiltinFunctionRepository repository) {
repository.register(castToFloat());
repository.register(castToDouble());
repository.register(castToBoolean());
repository.register(castToIp());
repository.register(castToDate());
repository.register(castToTime());
repository.register(castToTimestamp());
Expand Down Expand Up @@ -173,6 +176,13 @@ private static DefaultFunctionResolver castToBoolean() {
impl(nullMissingHandling((v) -> v), BOOLEAN, BOOLEAN));
}

private static DefaultFunctionResolver castToIp() {
return FunctionDSL.define(
BuiltinFunctionName.CAST_TO_IP.getName(),
impl(nullMissingHandling((v) -> v), IP, IP),
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ordering matters - make sure to but the naive cast last

impl(nullMissingHandling((v) -> new ExprIpValue(v.stringValue())), IP, STRING));
}

private static DefaultFunctionResolver castToDate() {
return FunctionDSL.define(
BuiltinFunctionName.CAST_TO_DATE.getName(),
Expand Down
Loading
Loading