Skip to content

Commit

Permalink
Add default value in denylist (#583) (#585)
Browse files Browse the repository at this point in the history 
Signed-off-by: Heemin Kim <[email protected]>
(cherry picked from commit e91b47c)

Co-authored-by: Heemin Kim <[email protected]>
  • Loading branch information
@opensearch-trigger-bot @heemin32
opensearch-trigger-bot[bot] and heemin32 authored Nov 6, 2023
1 parent 07fd490 commit 979f9a4
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 5 deletions.
28 changes: 26 additions & 2 deletions src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Collections;
import java.util.Arrays;
import java.util.List;
import java.util.function.Function;

Expand Down Expand Up @@ -77,10 +77,34 @@ public class Ip2GeoSettings {

/**
* A list of CIDR which will be blocked to be used as datasource endpoint
* Private network addresses will be blocked as default
*/
public static final Setting<List<String>> DATASOURCE_ENDPOINT_DENYLIST = Setting.listSetting(
"plugins.geospatial.ip2geo.datasource.endpoint.denylist",
Collections.emptyList(),
Arrays.asList(
"127.0.0.0/8",
"169.254.0.0/16",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"0.0.0.0/8",
"100.64.0.0/10",
"192.0.0.0/24",
"192.0.2.0/24",
"198.18.0.0/15",
"192.88.99.0/24",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"240.0.0.0/4",
"255.255.255.255/32",
"::1/128",
"fe80::/10",
"fc00::/7",
"::/128",
"2001:db8::/32",
"ff00::/8"
),
Function.identity(),
Setting.Property.NodeScope,
Setting.Property.Dynamic
Expand Down
14 changes: 14 additions & 0 deletions src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,20 @@ protected Map<String, Object> simulatePipeline(final String name, List<Object> d
return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map();
}

protected Response updateClusterSetting(final Map<String, Object> properties) throws IOException {
XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
builder.startObject("transient");
for (Map.Entry<String, Object> config : properties.entrySet()) {
builder.field(config.getKey(), config.getValue());
}
builder.endObject();
builder.endObject();

Request request = new Request(PUT, "/_cluster/settings");
request.setJsonEntity(builder.toString());
return client().performRequest(request);
}

protected static void createIndex(String name, Settings settings, Map<String, String> fieldMap) throws IOException {
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject(MAPPING_PROPERTIES_KEY);
for (Map.Entry<String, String> entry : fieldMap.entrySet()) {
Expand Down
5 changes: 4 additions & 1 deletion src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,12 @@

package org.opensearch.geospatial.ip2geo;

import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

import java.io.File;
import java.net.URL;
import java.nio.file.Paths;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
Expand Down Expand Up @@ -99,6 +101,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase {
protected Ip2GeoProcessorDao ip2GeoProcessorDao;
@Mock
protected RoutingTable routingTable;
@Mock
protected URLDenyListChecker urlDenyListChecker;
protected IngestMetadata ingestMetadata;
protected NoOpNodeClient client;
Expand All @@ -117,7 +120,7 @@ public void prepareIp2GeoTestCase() {
clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings()));
lockService = new LockService(client, clusterService);
ingestMetadata = new IngestMetadata(Collections.emptyMap());
urlDenyListChecker = spy(new URLDenyListChecker(clusterSettings));
when(urlDenyListChecker.toUrlIfNotInDenyList(anyString())).thenAnswer(i -> new URL(i.getArgument(0)));
when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata);
when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
Expand Down
48 changes: 48 additions & 0 deletions src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
Expand All @@ -16,10 +17,12 @@
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.opensearch.client.ResponseException;
import org.opensearch.common.settings.Settings;
import org.opensearch.core.rest.RestStatus;
import org.opensearch.geospatial.GeospatialRestTestCase;
import org.opensearch.geospatial.GeospatialTestHelper;
import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer;
import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;

public class UpdateDatasourceIT extends GeospatialRestTestCase {
// Use this value in resource name to avoid name conflict among tests
Expand All @@ -35,8 +38,50 @@ public static void stop() {
Ip2GeoDataServer.stop();
}

@SneakyThrows
public void testUpdateDatasource_whenPrivateNetwork_thenBlocked() {
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
try {
Map<String, Object> datasourceProperties = Map.of(
PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(),
Ip2GeoDataServer.getEndpointCountry()
);

// Create datasource and wait for it to be available
createDatasource(datasourceName, datasourceProperties);
isDatasourceCreated = true;
waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10));

// Revert deny list to its default value and private network ip should be blocked
updateClusterSetting(
Map.of(
Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(),
Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY)
)
);
int updateIntervalInDays = 1;
ResponseException exception = expectThrows(
ResponseException.class,
() -> updateDatasourceEndpoint(datasourceName, "http://127.0.0.1:9200/city/manifest_local.json", updateIntervalInDays)
);
assertEquals(400, exception.getResponse().getStatusLine().getStatusCode());
assertTrue(exception.getMessage().contains("blocked by deny list"));
} finally {
if (isDatasourceCreated) {
deleteDatasource(datasourceName, 3);
}
}
}

@SneakyThrows
public void testUpdateDatasource_whenValidInput_thenUpdated() {
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
try {
Expand Down Expand Up @@ -65,6 +110,9 @@ public void testUpdateDatasource_whenValidInput_thenUpdated() {

@SneakyThrows
public void testUpdateDatasource_whenIncompatibleFields_thenFails() {
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
try {
Expand Down
32 changes: 32 additions & 0 deletions src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,10 @@

package org.opensearch.geospatial.ip2geo.common;

import java.util.Arrays;
import java.util.List;

import org.opensearch.common.settings.Settings;
import org.opensearch.test.OpenSearchTestCase;

public class Ip2GeoSettingsTests extends OpenSearchTestCase {
Expand All @@ -18,4 +22,32 @@ public void testValidateValidUrl() {
Ip2GeoSettings.DatasourceEndpointValidator validator = new Ip2GeoSettings.DatasourceEndpointValidator();
validator.validate("https://test.com");
}

public void testDenyListDefaultValue() {
List<String> privateNetworks = Arrays.asList(
"127.0.0.0/8",
"169.254.0.0/16",
"10.0.0.0/8",
"172.16.0.0/12",
"192.168.0.0/16",
"0.0.0.0/8",
"100.64.0.0/10",
"192.0.0.0/24",
"192.0.2.0/24",
"198.18.0.0/15",
"192.88.99.0/24",
"198.51.100.0/24",
"203.0.113.0/24",
"224.0.0.0/4",
"240.0.0.0/4",
"255.255.255.255/32",
"::1/128",
"fe80::/10",
"fc00::/7",
"::/128",
"2001:db8::/32",
"ff00::/8"
);
assertEquals(privateNetworks, Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY));
}
}
33 changes: 31 additions & 2 deletions src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
import java.io.IOException;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
Expand All @@ -17,6 +18,8 @@

import lombok.SneakyThrows;

import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.opensearch.client.Response;
import org.opensearch.client.ResponseException;
import org.opensearch.common.Randomness;
Expand All @@ -25,6 +28,7 @@
import org.opensearch.geospatial.GeospatialTestHelper;
import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer;
import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest;
import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;

public class Ip2GeoProcessorIT extends GeospatialRestTestCase {
// Use this value in resource name to avoid name conflict among tests
Expand All @@ -34,9 +38,21 @@ public class Ip2GeoProcessorIT extends GeospatialRestTestCase {
private static final String IP = "ip";
private static final String SOURCE = "_source";

@BeforeClass
public static void start() {
Ip2GeoDataServer.start();
}

@AfterClass
public static void stop() {
Ip2GeoDataServer.stop();
}

@SneakyThrows
public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() {
Ip2GeoDataServer.start();
// Reset deny list to allow private network access during test
updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList()));

boolean isDatasourceCreated = false;
boolean isProcessorCreated = false;
String pipelineName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
Expand Down Expand Up @@ -109,13 +125,26 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() {
} catch (Exception e) {
exception = e;
}
Ip2GeoDataServer.stop();
if (exception != null) {
throw exception;
}
}
}

@SneakyThrows
public void testCreateIp2GeoProcessor_whenPrivateAddress_thenBlocked() {
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
Map<String, Object> datasourceProperties = Map.of(
PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(),
"http://127.0.0.1:9200/city/manifest_local.json"
);

// Create datasource and wait for it to be available
ResponseException exception = expectThrows(ResponseException.class, () -> createDatasource(datasourceName, datasourceProperties));
assertEquals(400, exception.getResponse().getStatusLine().getStatusCode());
assertTrue(exception.getMessage().contains("blocked by deny list"));
}

private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map<String, Object> properties) throws IOException {
String field = GeospatialTestHelper.randomLowerCaseString();
String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();
Expand Down

0 comments on commit 979f9a4

Please sign in to comment.