From 0504aa3c5fc86f14785b0a1e964a674693ca969b Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Mon, 6 Nov 2023 11:00:43 -0800 Subject: [PATCH] Add default value in denylist Signed-off-by: Heemin Kim --- .../ip2geo/common/Ip2GeoSettings.java | 28 ++++++++++- .../geospatial/GeospatialRestTestCase.java | 14 ++++++ .../geospatial/ip2geo/Ip2GeoTestCase.java | 5 +- .../ip2geo/action/UpdateDatasourceIT.java | 48 +++++++++++++++++++ .../ip2geo/common/Ip2GeoSettingsTests.java | 32 +++++++++++++ .../ip2geo/processor/Ip2GeoProcessorIT.java | 33 ++++++++++++- 6 files changed, 155 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index f48bb84c..142055ee 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -8,7 +8,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import java.util.function.Function; @@ -77,10 +77,34 @@ public class Ip2GeoSettings { /** * A list of CIDR which will be blocked to be used as datasource endpoint + * Private network addresses will be blocked as default */ public static final Setting> DATASOURCE_ENDPOINT_DENYLIST = Setting.listSetting( "plugins.geospatial.ip2geo.datasource.endpoint.denylist", - Collections.emptyList(), + Arrays.asList( + "127.0.0.0/8", + "169.254.0.0/16", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "0.0.0.0/8", + "100.64.0.0/10", + "192.0.0.0/24", + "192.0.2.0/24", + "198.18.0.0/15", + "192.88.99.0/24", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + "255.255.255.255/32", + "::1/128", + "fe80::/10", + "fc00::/7", + "::/128", + "2001:db8::/32", + "ff00::/8" + ), Function.identity(), Setting.Property.NodeScope, Setting.Property.Dynamic diff --git a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java index 62654f52..ea89b4c3 100644 --- a/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java +++ b/src/test/java/org/opensearch/geospatial/GeospatialRestTestCase.java @@ -193,6 +193,20 @@ protected Map simulatePipeline(final String name, List d return createParser(XContentType.JSON.xContent(), EntityUtils.toString(response.getEntity())).map(); } + protected Response updateClusterSetting(final Map properties) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder().startObject(); + builder.startObject("transient"); + for (Map.Entry config : properties.entrySet()) { + builder.field(config.getKey(), config.getValue()); + } + builder.endObject(); + builder.endObject(); + + Request request = new Request(PUT, "/_cluster/settings"); + request.setJsonEntity(builder.toString()); + return client().performRequest(request); + } + protected static void createIndex(String name, Settings settings, Map fieldMap) throws IOException { XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject(MAPPING_PROPERTIES_KEY); for (Map.Entry entry : fieldMap.entrySet()) { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index f16ed3e3..4d147d79 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -5,10 +5,12 @@ package org.opensearch.geospatial.ip2geo; +import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; import java.io.File; +import java.net.URL; import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; @@ -99,6 +101,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { protected Ip2GeoProcessorDao ip2GeoProcessorDao; @Mock protected RoutingTable routingTable; + @Mock protected URLDenyListChecker urlDenyListChecker; protected IngestMetadata ingestMetadata; protected NoOpNodeClient client; @@ -117,7 +120,7 @@ public void prepareIp2GeoTestCase() { clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings())); lockService = new LockService(client, clusterService); ingestMetadata = new IngestMetadata(Collections.emptyMap()); - urlDenyListChecker = spy(new URLDenyListChecker(clusterSettings)); + when(urlDenyListChecker.toUrlIfNotInDenyList(anyString())).thenAnswer(i -> new URL(i.getArgument(0))); when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); when(clusterService.getSettings()).thenReturn(Settings.EMPTY); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java index 6c7baa2f..12da9437 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/UpdateDatasourceIT.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.time.Duration; +import java.util.Collections; import java.util.List; import java.util.Locale; import java.util.Map; @@ -16,10 +17,12 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.opensearch.client.ResponseException; +import org.opensearch.common.settings.Settings; import org.opensearch.core.rest.RestStatus; import org.opensearch.geospatial.GeospatialRestTestCase; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; public class UpdateDatasourceIT extends GeospatialRestTestCase { // Use this value in resource name to avoid name conflict among tests @@ -35,8 +38,50 @@ public static void stop() { Ip2GeoDataServer.stop(); } + @SneakyThrows + public void testUpdateDatasource_whenPrivateNetwork_thenBlocked() { + // Reset deny list to allow private network access during test + updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList())); + + boolean isDatasourceCreated = false; + String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); + try { + Map datasourceProperties = Map.of( + PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(), + Ip2GeoDataServer.getEndpointCountry() + ); + + // Create datasource and wait for it to be available + createDatasource(datasourceName, datasourceProperties); + isDatasourceCreated = true; + waitForDatasourceToBeAvailable(datasourceName, Duration.ofSeconds(10)); + + // Revert deny list to its default value and private network ip should be blocked + updateClusterSetting( + Map.of( + Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), + Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY) + ) + ); + int updateIntervalInDays = 1; + ResponseException exception = expectThrows( + ResponseException.class, + () -> updateDatasourceEndpoint(datasourceName, "http://127.0.0.1:9200/city/manifest_local.json", updateIntervalInDays) + ); + assertEquals(400, exception.getResponse().getStatusLine().getStatusCode()); + assertTrue(exception.getMessage().contains("blocked by deny list")); + } finally { + if (isDatasourceCreated) { + deleteDatasource(datasourceName, 3); + } + } + } + @SneakyThrows public void testUpdateDatasource_whenValidInput_thenUpdated() { + // Reset deny list to allow private network access during test + updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList())); + boolean isDatasourceCreated = false; String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); try { @@ -65,6 +110,9 @@ public void testUpdateDatasource_whenValidInput_thenUpdated() { @SneakyThrows public void testUpdateDatasource_whenIncompatibleFields_thenFails() { + // Reset deny list to allow private network access during test + updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList())); + boolean isDatasourceCreated = false; String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); try { diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java index ffd40f5f..066c10c8 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettingsTests.java @@ -5,6 +5,10 @@ package org.opensearch.geospatial.ip2geo.common; +import java.util.Arrays; +import java.util.List; + +import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; public class Ip2GeoSettingsTests extends OpenSearchTestCase { @@ -18,4 +22,32 @@ public void testValidateValidUrl() { Ip2GeoSettings.DatasourceEndpointValidator validator = new Ip2GeoSettings.DatasourceEndpointValidator(); validator.validate("https://test.com"); } + + public void testDenyListDefaultValue() { + List privateNetworks = Arrays.asList( + "127.0.0.0/8", + "169.254.0.0/16", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + "0.0.0.0/8", + "100.64.0.0/10", + "192.0.0.0/24", + "192.0.2.0/24", + "198.18.0.0/15", + "192.88.99.0/24", + "198.51.100.0/24", + "203.0.113.0/24", + "224.0.0.0/4", + "240.0.0.0/4", + "255.255.255.255/32", + "::1/128", + "fe80::/10", + "fc00::/7", + "::/128", + "2001:db8::/32", + "ff00::/8" + ); + assertEquals(privateNetworks, Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.get(Settings.EMPTY)); + } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java index d73893f3..1259b5c7 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/processor/Ip2GeoProcessorIT.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.time.Duration; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -17,6 +18,8 @@ import lombok.SneakyThrows; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; import org.opensearch.common.Randomness; @@ -25,6 +28,7 @@ import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.Ip2GeoDataServer; import org.opensearch.geospatial.ip2geo.action.PutDatasourceRequest; +import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; public class Ip2GeoProcessorIT extends GeospatialRestTestCase { // Use this value in resource name to avoid name conflict among tests @@ -34,9 +38,21 @@ public class Ip2GeoProcessorIT extends GeospatialRestTestCase { private static final String IP = "ip"; private static final String SOURCE = "_source"; + @BeforeClass + public static void start() { + Ip2GeoDataServer.start(); + } + + @AfterClass + public static void stop() { + Ip2GeoDataServer.stop(); + } + @SneakyThrows public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { - Ip2GeoDataServer.start(); + // Reset deny list to allow private network access during test + updateClusterSetting(Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Collections.emptyList())); + boolean isDatasourceCreated = false; boolean isProcessorCreated = false; String pipelineName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); @@ -109,13 +125,26 @@ public void testCreateIp2GeoProcessor_whenValidInput_thenAddData() { } catch (Exception e) { exception = e; } - Ip2GeoDataServer.stop(); if (exception != null) { throw exception; } } } + @SneakyThrows + public void testCreateIp2GeoProcessor_whenPrivateAddress_thenBlocked() { + String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString(); + Map datasourceProperties = Map.of( + PutDatasourceRequest.ENDPOINT_FIELD.getPreferredName(), + "http://127.0.0.1:9200/city/manifest_local.json" + ); + + // Create datasource and wait for it to be available + ResponseException exception = expectThrows(ResponseException.class, () -> createDatasource(datasourceName, datasourceProperties)); + assertEquals(400, exception.getResponse().getStatusLine().getStatusCode()); + assertTrue(exception.getMessage().contains("blocked by deny list")); + } + private Response createIp2GeoProcessorPipeline(final String pipelineName, final Map properties) throws IOException { String field = GeospatialTestHelper.randomLowerCaseString(); String datasourceName = PREFIX + GeospatialTestHelper.randomLowerCaseString();