From 86e5a865aaee07320e28c4ae3cf66182946c0c0e Mon Sep 17 00:00:00 2001 From: Heemin Kim Date: Thu, 26 Oct 2023 21:43:34 -0700 Subject: [PATCH] Add denylist ip config for datasource endpoint Signed-off-by: Heemin Kim --- CHANGELOG.md | 1 + build.gradle | 1 + .../action/RestPutDatasourceHandler.java | 8 +- .../action/RestUpdateDatasourceHandler.java | 11 +++ .../ip2geo/common/Ip2GeoSettings.java | 15 +++- .../ip2geo/common/URLDenyListChecker.java | 63 ++++++++++++++++ .../geospatial/ip2geo/dao/GeoIpDataDao.java | 7 +- .../jobscheduler/DatasourceUpdateService.java | 8 +- .../geospatial/plugin/GeospatialPlugin.java | 16 +++- .../geospatial/ClusterSettingHelper.java | 64 ++++++++++++++++ .../geospatial/ip2geo/Ip2GeoTestCase.java | 3 + .../action/RestPutDatasourceHandlerTests.java | 22 +++++- .../RestUpdateDatasourceHandlerTests.java | 18 ++++- .../common/URLDenyListCheckerTests.java | 73 +++++++++++++++++++ .../ip2geo/dao/GeoIpDataDaoTests.java | 6 +- .../DatasourceUpdateServiceTests.java | 7 +- .../plugin/GeospatialPluginTests.java | 6 +- 17 files changed, 308 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/opensearch/geospatial/ip2geo/common/URLDenyListChecker.java create mode 100644 src/test/java/org/opensearch/geospatial/ClusterSettingHelper.java create mode 100644 src/test/java/org/opensearch/geospatial/ip2geo/common/URLDenyListCheckerTests.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b426889..1f692af4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on ho ## [Unreleased 2.x](https://github.com/opensearch-project/geospatial/compare/2.11...2.x) ### Features +* Add denylist ip config for datasource endpoint ([#573](https://github.com/opensearch-project/geospatial/pull/573)) ### Enhancements ### Bug Fixes ### Infrastructure diff --git a/build.gradle b/build.gradle index 3122b0df..d788f0f8 100644 --- a/build.gradle +++ b/build.gradle @@ -164,6 +164,7 @@ dependencies { implementation "org.apache.commons:commons-csv:1.10.0" zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}" compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}" + implementation "com.github.seancfoley:ipaddress:5.4.0" } licenseHeaders.enabled = true diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java index 0fac4ec9..8645d46b 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java @@ -17,6 +17,7 @@ import org.opensearch.common.unit.TimeValue; import org.opensearch.core.xcontent.XContentParser; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -38,9 +39,11 @@ public class RestPutDatasourceHandler extends BaseRestHandler { private static final String ACTION_NAME = "ip2geo_datasource_put"; private final ClusterSettings clusterSettings; + private final URLDenyListChecker urlDenyListChecker; - public RestPutDatasourceHandler(final ClusterSettings clusterSettings) { + public RestPutDatasourceHandler(final ClusterSettings clusterSettings, final URLDenyListChecker urlDenyListChecker) { this.clusterSettings = clusterSettings; + this.urlDenyListChecker = urlDenyListChecker; } @Override @@ -62,6 +65,9 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No if (putDatasourceRequest.getUpdateInterval() == null) { putDatasourceRequest.setUpdateInterval(TimeValue.timeValueDays(clusterSettings.get(Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL))); } + + // Call to validate if URL is in a deny-list or not. + urlDenyListChecker.toUrlIfNotInDenyList(putDatasourceRequest.getEndpoint()); return channel -> client.executeLocally(PutDatasourceAction.INSTANCE, putDatasourceRequest, new RestToXContentListener<>(channel)); } diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java index f9ba73ec..8ecc2cfc 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java @@ -14,6 +14,7 @@ import org.opensearch.client.node.NodeClient; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.rest.BaseRestHandler; import org.opensearch.rest.RestRequest; import org.opensearch.rest.action.RestToXContentListener; @@ -24,6 +25,12 @@ public class RestUpdateDatasourceHandler extends BaseRestHandler { private static final String ACTION_NAME = "ip2geo_datasource_update"; + private final URLDenyListChecker urlDenyListChecker; + + public RestUpdateDatasourceHandler(final URLDenyListChecker urlDenyListChecker) { + this.urlDenyListChecker = urlDenyListChecker; + } + @Override public String getName() { return ACTION_NAME; @@ -37,6 +44,10 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No UpdateDatasourceRequest.PARSER.parse(parser, updateDatasourceRequest, null); } } + if (updateDatasourceRequest.getEndpoint() != null) { + // Call to validate if URL is in a deny-list or not. + urlDenyListChecker.toUrlIfNotInDenyList(updateDatasourceRequest.getEndpoint()); + } return channel -> client.executeLocally( UpdateDatasourceAction.INSTANCE, updateDatasourceRequest, diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java index 16aba0e1..f48bb84c 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java @@ -8,7 +8,9 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.Collections; import java.util.List; +import java.util.function.Function; import org.opensearch.common.settings.Setting; import org.opensearch.common.unit.TimeValue; @@ -73,12 +75,23 @@ public class Ip2GeoSettings { Setting.Property.Dynamic ); + /** + * A list of CIDR which will be blocked to be used as datasource endpoint + */ + public static final Setting> DATASOURCE_ENDPOINT_DENYLIST = Setting.listSetting( + "plugins.geospatial.ip2geo.datasource.endpoint.denylist", + Collections.emptyList(), + Function.identity(), + Setting.Property.NodeScope, + Setting.Property.Dynamic + ); + /** * Return all settings of Ip2Geo feature * @return a list of all settings for Ip2Geo feature */ public static final List> settings() { - return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, TIMEOUT, CACHE_SIZE); + return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, TIMEOUT, CACHE_SIZE, DATASOURCE_ENDPOINT_DENYLIST); } /** diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/common/URLDenyListChecker.java b/src/main/java/org/opensearch/geospatial/ip2geo/common/URLDenyListChecker.java new file mode 100644 index 00000000..ab1e6f00 --- /dev/null +++ b/src/main/java/org/opensearch/geospatial/ip2geo/common/URLDenyListChecker.java @@ -0,0 +1,63 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.UnknownHostException; +import java.util.List; + +import lombok.extern.log4j.Log4j2; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.settings.ClusterSettings; + +import inet.ipaddr.IPAddressString; + +/** + * A class to check url against a deny-list + */ +@Log4j2 +public class URLDenyListChecker { + private final ClusterSettings clusterSettings; + + public URLDenyListChecker(final ClusterSettings clusterSettings) { + this.clusterSettings = clusterSettings; + } + + /** + * Convert String to URL after verifying the url is not on a deny-list + * + * @param url value to validate and convert to URL + * @return value in URL type + */ + public URL toUrlIfNotInDenyList(final String url) { + try { + return toUrlIfNotInDenyList(url, clusterSettings.get(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST)); + } catch (UnknownHostException e) { + log.error("Unknown host", e); + throw new IllegalArgumentException("host is unknown"); + } catch (MalformedURLException e) { + log.error("Malformed URL", e); + throw new IllegalArgumentException("url is malformed"); + } + } + + @SuppressForbidden(reason = "Need to connect to http endpoint to read GeoIP database file") + private URL toUrlIfNotInDenyList(final String url, final List denyList) throws UnknownHostException, MalformedURLException { + URL urlToReturn = new URL(url); + if (isInDenyList(new IPAddressString(InetAddress.getByName(urlToReturn.getHost()).getHostAddress()), denyList)) { + throw new IllegalArgumentException( + "given endpoint is blocked by deny list in cluster setting " + Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey() + ); + } + return urlToReturn; + } + + private boolean isInDenyList(final IPAddressString url, final List denyList) { + return denyList.stream().map(cidr -> new IPAddressString(cidr)).anyMatch(cidr -> cidr.contains(url)); + } +} diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java index 47774b90..e9ad161a 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java @@ -61,6 +61,7 @@ import org.opensearch.geospatial.constants.IndexSetting; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.geospatial.shared.Constants; import org.opensearch.geospatial.shared.StashedThreadContext; import org.opensearch.index.query.QueryBuilders; @@ -91,11 +92,13 @@ public class GeoIpDataDao { private final ClusterService clusterService; private final ClusterSettings clusterSettings; private final Client client; + private final URLDenyListChecker urlDenyListChecker; - public GeoIpDataDao(final ClusterService clusterService, final Client client) { + public GeoIpDataDao(final ClusterService clusterService, final Client client, final URLDenyListChecker urlDenyListChecker) { this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); this.client = client; + this.urlDenyListChecker = urlDenyListChecker; } /** @@ -172,7 +175,7 @@ public CSVParser getDatabaseReader(final DatasourceManifest manifest) { SpecialPermission.check(); return AccessController.doPrivileged((PrivilegedAction) () -> { try { - URL zipUrl = new URL(manifest.getUrl()); + URL zipUrl = urlDenyListChecker.toUrlIfNotInDenyList(manifest.getUrl()); return internalGetDatabaseReader(manifest, zipUrl.openConnection()); } catch (IOException e) { throw new OpenSearchException("failed to read geoip data from {}", manifest.getUrl(), e); diff --git a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java index 2abf8a79..1696944f 100644 --- a/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java +++ b/src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java @@ -25,6 +25,7 @@ import org.opensearch.geospatial.annotation.VisibleForTesting; import org.opensearch.geospatial.ip2geo.common.DatasourceManifest; import org.opensearch.geospatial.ip2geo.common.DatasourceState; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; @@ -37,16 +38,19 @@ public class DatasourceUpdateService { private final ClusterSettings clusterSettings; private final DatasourceDao datasourceDao; private final GeoIpDataDao geoIpDataDao; + private final URLDenyListChecker urlDenyListChecker; public DatasourceUpdateService( final ClusterService clusterService, final DatasourceDao datasourceDao, - final GeoIpDataDao geoIpDataDao + final GeoIpDataDao geoIpDataDao, + final URLDenyListChecker urlDenyListChecker ) { this.clusterService = clusterService; this.clusterSettings = clusterService.getClusterSettings(); this.datasourceDao = datasourceDao; this.geoIpDataDao = geoIpDataDao; + this.urlDenyListChecker = urlDenyListChecker; } /** @@ -61,7 +65,7 @@ public DatasourceUpdateService( * @throws IOException */ public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable renewLock) throws IOException { - URL url = new URL(datasource.getEndpoint()); + URL url = urlDenyListChecker.toUrlIfNotInDenyList(datasource.getEndpoint()); DatasourceManifest manifest = DatasourceManifest.Builder.build(url); if (shouldUpdate(datasource, manifest) == false) { diff --git a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java index e38e846d..e8e3247a 100644 --- a/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java +++ b/src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java @@ -54,6 +54,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao; @@ -97,6 +98,7 @@ public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlug private Ip2GeoCachedDao ip2GeoCachedDao; private DatasourceDao datasourceDao; private GeoIpDataDao geoIpDataDao; + private URLDenyListChecker urlDenyListChecker; @Override public Collection getSystemIndexDescriptors(Settings settings) { @@ -105,8 +107,9 @@ public Collection getSystemIndexDescriptors(Settings sett @Override public Map getProcessors(Processor.Parameters parameters) { + this.urlDenyListChecker = new URLDenyListChecker(parameters.ingestService.getClusterService().getClusterSettings()); this.datasourceDao = new DatasourceDao(parameters.client, parameters.ingestService.getClusterService()); - this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client); + this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client, urlDenyListChecker); this.ip2GeoCachedDao = new Ip2GeoCachedDao(parameters.ingestService.getClusterService(), datasourceDao, geoIpDataDao); return MapBuilder.newMapBuilder() .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory()) @@ -153,7 +156,12 @@ public Collection createComponents( IndexNameExpressionResolver indexNameExpressionResolver, Supplier repositoriesServiceSupplier ) { - DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao); + DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService( + clusterService, + datasourceDao, + geoIpDataDao, + urlDenyListChecker + ); Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool); Ip2GeoLockService ip2GeoLockService = new Ip2GeoLockService(clusterService, client); /** @@ -187,9 +195,9 @@ public List getRestHandlers( List geoJsonHandlers = List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction()); List ip2geoHandlers = List.of( - new RestPutDatasourceHandler(clusterSettings), + new RestPutDatasourceHandler(clusterSettings, urlDenyListChecker), new RestGetDatasourceHandler(), - new RestUpdateDatasourceHandler(), + new RestUpdateDatasourceHandler(urlDenyListChecker), new RestDeleteDatasourceHandler() ); diff --git a/src/test/java/org/opensearch/geospatial/ClusterSettingHelper.java b/src/test/java/org/opensearch/geospatial/ClusterSettingHelper.java new file mode 100644 index 00000000..93bde1b6 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ClusterSettingHelper.java @@ -0,0 +1,64 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial; + +import static org.apache.lucene.tests.util.LuceneTestCase.createTempDir; +import static org.opensearch.test.NodeRoles.dataNode; +import static org.opensearch.test.OpenSearchTestCase.getTestTransportPlugin; +import static org.opensearch.test.OpenSearchTestCase.getTestTransportType; +import static org.opensearch.test.OpenSearchTestCase.randomLong; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.opensearch.cluster.ClusterName; +import org.opensearch.common.SuppressForbidden; +import org.opensearch.common.network.NetworkModule; +import org.opensearch.common.settings.Settings; +import org.opensearch.env.Environment; +import org.opensearch.geospatial.plugin.GeospatialPlugin; +import org.opensearch.node.MockNode; +import org.opensearch.node.Node; +import org.opensearch.plugins.Plugin; +import org.opensearch.test.InternalTestCluster; +import org.opensearch.test.MockHttpTransport; + +@SuppressForbidden(reason = "used only for testing") +public class ClusterSettingHelper { + public Node createMockNode(Map configSettings) throws IOException { + Path configDir = createTempDir(); + File configFile = configDir.resolve("opensearch.yml").toFile(); + FileWriter configFileWriter = new FileWriter(configFile); + + for (Map.Entry setting : configSettings.entrySet()) { + configFileWriter.write("\"" + setting.getKey() + "\": " + setting.getValue()); + } + configFileWriter.close(); + return new MockNode(baseSettings().build(), basePlugins(), configDir, true); + } + + private List> basePlugins() { + List> plugins = new ArrayList<>(); + plugins.add(getTestTransportPlugin()); + plugins.add(MockHttpTransport.TestPlugin.class); + plugins.add(GeospatialPlugin.class); + return plugins; + } + + private static Settings.Builder baseSettings() { + final Path tempDir = createTempDir(); + return Settings.builder() + .put(ClusterName.CLUSTER_NAME_SETTING.getKey(), InternalTestCluster.clusterName("single-node-cluster", randomLong())) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .put(NetworkModule.TRANSPORT_TYPE_KEY, getTestTransportType()) + .put(dataNode()); + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java index b70b3f03..f16ed3e3 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java @@ -47,6 +47,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao; @@ -98,6 +99,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase { protected Ip2GeoProcessorDao ip2GeoProcessorDao; @Mock protected RoutingTable routingTable; + protected URLDenyListChecker urlDenyListChecker; protected IngestMetadata ingestMetadata; protected NoOpNodeClient client; protected VerifyingClient verifyingClient; @@ -115,6 +117,7 @@ public void prepareIp2GeoTestCase() { clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings())); lockService = new LockService(client, clusterService); ingestMetadata = new IngestMetadata(Collections.emptyMap()); + urlDenyListChecker = spy(new URLDenyListChecker(clusterSettings)); when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata); when(clusterService.getSettings()).thenReturn(Settings.EMPTY); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java index ecbba865..7d770df6 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java @@ -5,6 +5,8 @@ package org.opensearch.geospatial.ip2geo.action; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; @@ -12,6 +14,8 @@ import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; +import lombok.SneakyThrows; + import org.junit.Before; import org.opensearch.common.SuppressForbidden; import org.opensearch.common.settings.ClusterSettings; @@ -21,6 +25,7 @@ import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.geospatial.GeospatialTestHelper; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.rest.RestRequest; import org.opensearch.test.rest.FakeRestRequest; import org.opensearch.test.rest.RestActionTestCase; @@ -29,17 +34,22 @@ public class RestPutDatasourceHandlerTests extends RestActionTestCase { private String path; private RestPutDatasourceHandler action; + private URLDenyListChecker urlDenyListChecker; @Before public void setupAction() { - action = new RestPutDatasourceHandler(new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings()))); + ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings())); + urlDenyListChecker = mock(URLDenyListChecker.class); + action = new RestPutDatasourceHandler(clusterSettings, urlDenyListChecker); controller().registerHandler(action); path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/%s"); } + @SneakyThrows public void testPrepareRequest() { + String endpoint = "https://test.com"; String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - String content = "{\"endpoint\":\"https://test.com\", \"update_interval_in_days\":1}"; + String content = String.format(Locale.ROOT, "{\"endpoint\":\"%s\", \"update_interval_in_days\":1}", endpoint); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) .withPath(String.format(Locale.ROOT, path, datasourceName)) .withContent(new BytesArray(content), XContentType.JSON) @@ -49,7 +59,7 @@ public void testPrepareRequest() { verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof PutDatasourceRequest); PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; - assertEquals("https://test.com", putDatasourceRequest.getEndpoint()); + assertEquals(endpoint, putDatasourceRequest.getEndpoint()); assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateInterval()); assertEquals(datasourceName, putDatasourceRequest.getName()); isExecuted.set(true); @@ -58,9 +68,12 @@ public void testPrepareRequest() { dispatchRequest(request); assertTrue(isExecuted.get()); + verify(urlDenyListChecker).toUrlIfNotInDenyList(endpoint); } + @SneakyThrows public void testPrepareRequestDefaultValue() { + String endpoint = "https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json"; String datasourceName = GeospatialTestHelper.randomLowerCaseString(); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) .withPath(String.format(Locale.ROOT, path, datasourceName)) @@ -70,7 +83,7 @@ public void testPrepareRequestDefaultValue() { verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> { assertTrue(actionRequest instanceof PutDatasourceRequest); PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest; - assertEquals("https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json", putDatasourceRequest.getEndpoint()); + assertEquals(endpoint, putDatasourceRequest.getEndpoint()); assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateInterval()); assertEquals(datasourceName, putDatasourceRequest.getName()); isExecuted.set(true); @@ -79,5 +92,6 @@ public void testPrepareRequestDefaultValue() { dispatchRequest(request); assertTrue(isExecuted.get()); + verify(urlDenyListChecker).toUrlIfNotInDenyList(endpoint); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java index ef15d030..284c160d 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandlerTests.java @@ -5,17 +5,24 @@ package org.opensearch.geospatial.ip2geo.action; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER; import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; +import lombok.SneakyThrows; + import org.junit.Before; import org.opensearch.common.unit.TimeValue; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.common.bytes.BytesArray; import org.opensearch.geospatial.GeospatialTestHelper; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.rest.RestRequest; import org.opensearch.test.rest.FakeRestRequest; import org.opensearch.test.rest.RestActionTestCase; @@ -23,17 +30,21 @@ public class RestUpdateDatasourceHandlerTests extends RestActionTestCase { private String path; private RestUpdateDatasourceHandler handler; + private URLDenyListChecker urlDenyListChecker; @Before public void setupAction() { - handler = new RestUpdateDatasourceHandler(); + urlDenyListChecker = mock(URLDenyListChecker.class); + handler = new RestUpdateDatasourceHandler(urlDenyListChecker); controller().registerHandler(handler); path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/%s/_settings"); } + @SneakyThrows public void testPrepareRequest_whenValidInput_thenSucceed() { + String endpoint = "https://test.com"; String datasourceName = GeospatialTestHelper.randomLowerCaseString(); - String content = "{\"endpoint\":\"https://test.com\", \"update_interval_in_days\":1}"; + String content = String.format(Locale.ROOT, "{\"endpoint\":\"%s\", \"update_interval_in_days\":1}", endpoint); RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT) .withPath(String.format(Locale.ROOT, path, datasourceName)) .withContent(new BytesArray(content), XContentType.JSON) @@ -52,8 +63,10 @@ public void testPrepareRequest_whenValidInput_thenSucceed() { dispatchRequest(request); assertTrue(isExecuted.get()); + verify(urlDenyListChecker).toUrlIfNotInDenyList(endpoint); } + @SneakyThrows public void testPrepareRequest_whenNullInput_thenSucceed() { String datasourceName = GeospatialTestHelper.randomLowerCaseString(); String content = "{}"; @@ -75,5 +88,6 @@ public void testPrepareRequest_whenNullInput_thenSucceed() { dispatchRequest(request); assertTrue(isExecuted.get()); + verify(urlDenyListChecker, never()).toUrlIfNotInDenyList(anyString()); } } diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/common/URLDenyListCheckerTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/common/URLDenyListCheckerTests.java new file mode 100644 index 00000000..7a35dfd8 --- /dev/null +++ b/src/test/java/org/opensearch/geospatial/ip2geo/common/URLDenyListCheckerTests.java @@ -0,0 +1,73 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.geospatial.ip2geo.common; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Map; + +import lombok.SneakyThrows; + +import org.junit.Before; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.Randomness; +import org.opensearch.geospatial.ClusterSettingHelper; +import org.opensearch.node.Node; +import org.opensearch.test.OpenSearchTestCase; + +public class URLDenyListCheckerTests extends OpenSearchTestCase { + private ClusterSettingHelper clusterSettingHelper; + + @Before + public void init() { + clusterSettingHelper = new ClusterSettingHelper(); + } + + @SneakyThrows + public void testToUrlIfNotInDenyListWithBlockedAddress() { + Node mockNode = clusterSettingHelper.createMockNode( + Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Arrays.asList("127.0.0.0/8")) + ); + mockNode.start(); + try { + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + URLDenyListChecker urlDenyListChecker = new URLDenyListChecker(clusterService.getClusterSettings()); + String endpoint = String.format( + Locale.ROOT, + "https://127.%d.%d.%d/v1/manifest.json", + Randomness.get().nextInt(256), + Randomness.get().nextInt(256), + Randomness.get().nextInt(256) + ); + expectThrows(IllegalArgumentException.class, () -> urlDenyListChecker.toUrlIfNotInDenyList(endpoint)); + } finally { + mockNode.close(); + } + } + + @SneakyThrows + public void testToUrlIfNotInDenyListWithNonBlockedAddress() { + Node mockNode = clusterSettingHelper.createMockNode( + Map.of(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey(), Arrays.asList("127.0.0.0/8")) + ); + mockNode.start(); + try { + ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); + URLDenyListChecker urlDenyListChecker = new URLDenyListChecker(clusterService.getClusterSettings()); + String endpoint = String.format( + Locale.ROOT, + "https://128.%d.%d.%d/v1/manifest.json", + Randomness.get().nextInt(256), + Randomness.get().nextInt(256), + Randomness.get().nextInt(256) + ); + // Expect no exception + urlDenyListChecker.toUrlIfNotInDenyList(endpoint); + } finally { + mockNode.close(); + } + } +} diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java index 3a985db1..247bff72 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDaoTests.java @@ -61,8 +61,8 @@ public class GeoIpDataDaoTests extends Ip2GeoTestCase { @Before public void init() { - noOpsGeoIpDataDao = new GeoIpDataDao(clusterService, client); - verifyingGeoIpDataDao = new GeoIpDataDao(clusterService, verifyingClient); + noOpsGeoIpDataDao = new GeoIpDataDao(clusterService, client, urlDenyListChecker); + verifyingGeoIpDataDao = new GeoIpDataDao(clusterService, verifyingClient, urlDenyListChecker); } public void testCreateIndexIfNotExistsWithExistingIndex() { @@ -131,6 +131,7 @@ public void testGetDatabaseReader() throws Exception { assertArrayEquals(expectedHeader, parser.iterator().next().values()); String[] expectedValues = { "1.0.0.0/24", "Australia" }; assertArrayEquals(expectedValues, parser.iterator().next().values()); + verify(urlDenyListChecker).toUrlIfNotInDenyList(manifest.getUrl()); } public void testGetDatabaseReaderNoFile() throws Exception { @@ -145,6 +146,7 @@ public void testGetDatabaseReaderNoFile() throws Exception { ); Exception exception = expectThrows(IllegalArgumentException.class, () -> noOpsGeoIpDataDao.getDatabaseReader(manifest)); assertTrue(exception.getMessage().contains("does not exist")); + verify(urlDenyListChecker).toUrlIfNotInDenyList(manifest.getUrl()); } @SneakyThrows diff --git a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java index 5e08400b..818b412f 100644 --- a/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java +++ b/src/test/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateServiceTests.java @@ -43,7 +43,7 @@ public class DatasourceUpdateServiceTests extends Ip2GeoTestCase { @Before public void init() { - datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao); + datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao, urlDenyListChecker); } @SneakyThrows @@ -64,6 +64,7 @@ public void testUpdateOrCreateGeoIpData_whenHashValueIsSame_thenSkipUpdate() { // Verify assertNotNull(datasource.getUpdateStats().getLastSkippedAt()); verify(datasourceDao).updateDatasource(datasource); + verify(urlDenyListChecker).toUrlIfNotInDenyList(datasource.getEndpoint()); } @SneakyThrows @@ -87,6 +88,7 @@ public void testUpdateOrCreateGeoIpData_whenExpired_thenUpdate() { // Verify verify(geoIpDataDao).putGeoIpData(eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), any(Runnable.class)); + verify(urlDenyListChecker).toUrlIfNotInDenyList(datasource.getEndpoint()); } @SneakyThrows @@ -108,6 +110,7 @@ public void testUpdateOrCreateGeoIpData_whenInvalidData_thenThrowException() { // Run expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class))); + verify(urlDenyListChecker).toUrlIfNotInDenyList(datasource.getEndpoint()); } @SneakyThrows @@ -127,6 +130,7 @@ public void testUpdateOrCreateGeoIpData_whenIncompatibleFields_thenThrowExceptio // Run expectThrows(OpenSearchException.class, () -> datasourceUpdateService.updateOrCreateGeoIpData(datasource, mock(Runnable.class))); + verify(urlDenyListChecker).toUrlIfNotInDenyList(datasource.getEndpoint()); } @SneakyThrows @@ -161,6 +165,7 @@ public void testUpdateOrCreateGeoIpData_whenValidInput_thenSucceed() { assertNotNull(datasource.getUpdateStats().getLastProcessingTimeInMillis()); verify(datasourceDao, times(2)).updateDatasource(datasource); verify(geoIpDataDao).putGeoIpData(eq(datasource.currentIndexName()), isA(String[].class), any(Iterator.class), any(Runnable.class)); + verify(urlDenyListChecker).toUrlIfNotInDenyList(datasource.getEndpoint()); } public void testWaitUntilAllShardsStarted_whenTimedOut_thenThrowException() { diff --git a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java index 09d5a4a8..48b23197 100644 --- a/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java +++ b/src/test/java/org/opensearch/geospatial/plugin/GeospatialPluginTests.java @@ -40,6 +40,7 @@ import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor; import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService; import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings; +import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker; import org.opensearch.geospatial.ip2geo.dao.DatasourceDao; import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao; import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao; @@ -63,12 +64,13 @@ public class GeospatialPluginTests extends OpenSearchTestCase { private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings())); + private final URLDenyListChecker urlDenyListChecker = new URLDenyListChecker(clusterSettings); private final List SUPPORTED_REST_HANDLERS = List.of( new RestUploadGeoJSONAction(), new RestUploadStatsAction(), - new RestPutDatasourceHandler(clusterSettings), + new RestPutDatasourceHandler(clusterSettings, urlDenyListChecker), new RestGetDatasourceHandler(), - new RestUpdateDatasourceHandler(), + new RestUpdateDatasourceHandler(urlDenyListChecker), new RestDeleteDatasourceHandler() );