Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.11] Add default value in denylist #585

Merged
merged 1 commit into from
Nov 6, 2023
Merged

[Backport 2.11] Add default value in denylist #585

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Collections;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;

Expand Down Expand Up @@ -77,10 +77,34 @@ public class Ip2GeoSettings {

/**
* A list of CIDR which will be blocked to be used as datasource endpoint
* Private network addresses will be blocked as default
*/
public static final Setting<List<String>> DATASOURCE_ENDPOINT_DENYLIST = Setting.listSetting(
"plugins.geospatial.ip2geo.datasource.endpoint.denylist",
Collections.emptyList(),
Arrays.asList(
"127.0.0.0/8",
"169.254.0.0/16",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"0.0.0.0/8",
"100.64.0.0/10",
"192.0.0.0/24",
"192.0.2.0/24",
"198.18.0.0/15",
"192.88.99.0/24",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"240.0.0.0/4",
"255.255.255.255/32",
"::1/128",
"fe80::/10",
"fc00::/7",
"::/128",
"2001:db8::/32",
"ff00::/8"
),
Function.identity(),
Setting.Property.NodeScope,
Setting.Property.Dynamic
Expand Down
14 changes: 14 additions & 0 deletions src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,20 @@ protected Map<String, Object> simulatePipeline(final String name, List<Object> d
return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map();
}

protected Response updateClusterSetting(final Map<String, Object> properties) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
builder.startObject("transient");
for (Map.Entry<String, Object> config : properties.entrySet()) {
builder.field(config.getKey(), config.getValue());
}
builder.endObject();
builder.endObject();

Request request = new Request(PUT, "/_cluster/settings");
request.setJsonEntity(builder.toString());
return client().performRequest(request);
}

protected static void createIndex(String name, Settings settings, Map<String, String> fieldMap) throws IOException {
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject(MAPPING_PROPERTIES_KEY);
for (Map.Entry<String, String> entry : fieldMap.entrySet()) {
Expand Down
5 changes: 4 additions & 1 deletion src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@

package org.opensearch.geospatial.ip2geo;

import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

import java.io.File;
import java.net.URL;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
Expand Down Expand Up @@ -99,6 +101,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase {
protected Ip2GeoProcessorDao ip2GeoProcessorDao;
@Mock
protected RoutingTable routingTable;
@Mock
protected URLDenyListChecker urlDenyListChecker;
protected IngestMetadata ingestMetadata;
protected NoOpNodeClient client;
Expand All @@ -117,7 +120,7 @@ public void prepareIp2GeoTestCase() {
clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings()));
lockService = new LockService(client, clusterService);
ingestMetadata = new IngestMetadata(Collections.emptyMap());
urlDenyListChecker = spy(new URLDenyListChecker(clusterSettings));
when(urlDenyListChecker.toUrlIfNotInDenyList(anyString())).thenAnswer(i -> new URL(i.getArgument(0)));
when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata);
when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
Expand Down
48 changes: 48 additions & 0 deletions src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand All @@ -16,10 +17,12 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.opensearch.client.ResponseException;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.geospatial.GeospatialRestTestCase;
import org.opensearch.geospatial.GeospatialTestHelper;
import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer;
import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;

public class UpdateDatasourceIT extends GeospatialRestTestCase {
// Use this value in resource name to avoid name conflict among tests
Expand All @@ -35,8 +38,50 @@ public static void stop() {
Ip2GeoDataServer.stop();
}

@SneakyThrows
public void testUpdateDatasource_whenPrivateNetwork_thenBlocked() {
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
try {
Map<String, Object> datasourceProperties = Map.of(
PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(),
Ip2GeoDataServer.getEndpointCountry()
);

// Create datasource and wait for it to be available
createDatasource(datasourceName, datasourceProperties);
isDatasourceCreated = true;
waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10));

// Revert deny list to its default value and private network ip should be blocked
updateClusterSetting(
Map.of(
Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(),
Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY)
)
);
int updateIntervalInDays = 1;
ResponseException exception = expectThrows(
ResponseException.class,
() -> updateDatasourceEndpoint(datasourceName, "http://127.0.0.1:9200/city/manifest_local.json", updateIntervalInDays)
);
assertEquals(400, exception.getResponse().getStatusLine().getStatusCode());
assertTrue(exception.getMessage().contains("blocked by deny list"));
} finally {
if (isDatasourceCreated) {
deleteDatasource(datasourceName, 3);
}
}
}

@SneakyThrows
public void testUpdateDatasource_whenValidInput_thenUpdated() {
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
try {
Expand Down Expand Up @@ -65,6 +110,9 @@ public void testUpdateDatasource_whenValidInput_thenUpdated() {

@SneakyThrows
public void testUpdateDatasource_whenIncompatibleFields_thenFails() {
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
try {
Expand Down
32 changes: 32 additions & 0 deletions src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@

package org.opensearch.geospatial.ip2geo.common;

import java.util.Arrays;
import java.util.List;

import org.opensearch.common.settings.Settings;
import org.opensearch.test.OpenSearchTestCase;

public class Ip2GeoSettingsTests extends OpenSearchTestCase {
Expand All @@ -18,4 +22,32 @@ public void testValidateValidUrl() {
Ip2GeoSettings.DatasourceEndpointValidator validator = new Ip2GeoSettings.DatasourceEndpointValidator();
validator.validate("https://test.com");
}

public void testDenyListDefaultValue() {
List<String> privateNetworks = Arrays.asList(
"127.0.0.0/8",
"169.254.0.0/16",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"0.0.0.0/8",
"100.64.0.0/10",
"192.0.0.0/24",
"192.0.2.0/24",
"198.18.0.0/15",
"192.88.99.0/24",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"240.0.0.0/4",
"255.255.255.255/32",
"::1/128",
"fe80::/10",
"fc00::/7",
"::/128",
"2001:db8::/32",
"ff00::/8"
);
assertEquals(privateNetworks, Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY));
}
}
33 changes: 31 additions & 2 deletions src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.io.IOException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
Expand All @@ -17,6 +18,8 @@

import lombok.SneakyThrows;

import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.opensearch.client.Response;
import org.opensearch.client.ResponseException;
import org.opensearch.common.Randomness;
Expand All @@ -25,6 +28,7 @@
import org.opensearch.geospatial.GeospatialTestHelper;
import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer;
import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest;
import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;

public class Ip2GeoProcessorIT extends GeospatialRestTestCase {
// Use this value in resource name to avoid name conflict among tests
Expand All @@ -34,9 +38,21 @@ public class Ip2GeoProcessorIT extends GeospatialRestTestCase {
private static final String IP = "ip";
private static final String SOURCE = "_source";

@BeforeClass
public static void start() {
Ip2GeoDataServer.start();
}

@AfterClass
public static void stop() {
Ip2GeoDataServer.stop();
}

@SneakyThrows
public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() {
Ip2GeoDataServer.start();
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
boolean isProcessorCreated = false;
String pipelineName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
Expand Down Expand Up @@ -109,13 +125,26 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() {
} catch (Exception e) {
exception = e;
}
Ip2GeoDataServer.stop();
if (exception != null) {
throw exception;
}
}
}

@SneakyThrows
public void testCreateIp2GeoProcessor_whenPrivateAddress_thenBlocked() {
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
Map<String, Object> datasourceProperties = Map.of(
PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(),
"http://127.0.0.1:9200/city/manifest_local.json"
);

// Create datasource and wait for it to be available
ResponseException exception = expectThrows(ResponseException.class, () -> createDatasource(datasourceName, datasourceProperties));
assertEquals(400, exception.getResponse().getStatusLine().getStatusCode());
assertTrue(exception.getMessage().contains("blocked by deny list"));
}

private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map<String, Object> properties) throws IOException {
String field = GeospatialTestHelper.randomLowerCaseString();
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
Expand Down
Loading