Add denylist ip config for datasource endpoint

Signed-off-by: Heemin Kim <[email protected]>

build.gradle

@@ -164,6 +164,9 @@ dependencies {
     implementation "org.apache.commons:commons-csv:1.10.0"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}"
     compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
+
+    // This is used inside EndpointManager for AOS only
+    implementation "com.github.seancfoley:ipaddress:5.4.0"
 }
 
 licenseHeaders.enabled = true

src/main/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandler.java

@@ -17,6 +17,7 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.RestToXContentListener;
@@ -38,9 +39,11 @@
 public class RestPutDatasourceHandler extends BaseRestHandler {
     private static final String ACTION_NAME = "ip2geo_datasource_put";
     private final ClusterSettings clusterSettings;
+    private final URLDenyListChecker urlDenyListChecker;
 
-    public RestPutDatasourceHandler(final ClusterSettings clusterSettings) {
+    public RestPutDatasourceHandler(final ClusterSettings clusterSettings, final URLDenyListChecker urlDenyListChecker) {
         this.clusterSettings = clusterSettings;
+        this.urlDenyListChecker = urlDenyListChecker;
     }
 
     @Override
@@ -62,6 +65,9 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No
         if (putDatasourceRequest.getUpdateInterval() == null) {
             putDatasourceRequest.setUpdateInterval(TimeValue.timeValueDays(clusterSettings.get(Ip2GeoSettings.DATASOURCE_UPDATE_INTERVAL)));
         }
+
+        // Call to validate if URL is in a deny-list or not.
+        urlDenyListChecker.toUrlIfNotInDenyList(putDatasourceRequest.getEndpoint());
         return channel -> client.executeLocally(PutDatasourceAction.INSTANCE, putDatasourceRequest, new RestToXContentListener<>(channel));
     }
 

src/main/java/org/opensearch/geospatial/ip2geo/action/RestUpdateDatasourceHandler.java

@@ -14,6 +14,7 @@
 
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.RestToXContentListener;
@@ -24,6 +25,12 @@
 public class RestUpdateDatasourceHandler extends BaseRestHandler {
     private static final String ACTION_NAME = "ip2geo_datasource_update";
 
+    private final URLDenyListChecker urlDenyListChecker;
+
+    public RestUpdateDatasourceHandler(final URLDenyListChecker urlDenyListChecker) {
+        this.urlDenyListChecker = urlDenyListChecker;
+    }
+
     @Override
     public String getName() {
         return ACTION_NAME;
@@ -37,6 +44,10 @@ protected RestChannelConsumer prepareRequest(final RestRequest request, final No
                 UpdateDatasourceRequest.PARSER.parse(parser, updateDatasourceRequest, null);
             }
         }
+        if (updateDatasourceRequest.getEndpoint() != null) {
+            // Call to validate if URL is in a deny-list or not.
+            urlDenyListChecker.toUrlIfNotInDenyList(updateDatasourceRequest.getEndpoint());
+        }
         return channel -> client.executeLocally(
             UpdateDatasourceAction.INSTANCE,
             updateDatasourceRequest,

src/main/java/org/opensearch/geospatial/ip2geo/common/Ip2GeoSettings.java

@@ -8,7 +8,10 @@
 import java.net.MalformedURLException;
 import java.net.URISyntaxException;
 import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
+import java.util.function.Function;
 
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.unit.TimeValue;
@@ -73,12 +76,23 @@ public class Ip2GeoSettings {
         Setting.Property.Dynamic
     );
 
+    /**
+     * A list of CIDR which will be blocked to be used as datasource endpoint
+     */
+    public static final Setting<List<String>> DATASOURCE_ENDPOINT_DENYLIST =
+        Setting.listSetting(
+            "plugins.geospatial.ip2geo.datasource.endpoint.denylist",
+            new ArrayList<>(),
+            Function.identity(),
+            Setting.Property.NodeScope,
+            Setting.Property.Dynamic);
+
     /**
      * Return all settings of Ip2Geo feature
      * @return a list of all settings for Ip2Geo feature
      */
     public static final List<Setting<?>> settings() {
-        return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, TIMEOUT, CACHE_SIZE);
+        return List.of(DATASOURCE_ENDPOINT, DATASOURCE_UPDATE_INTERVAL, BATCH_SIZE, TIMEOUT, CACHE_SIZE, DATASOURCE_ENDPOINT_DENYLIST);
     }
 
     /**

src/main/java/org/opensearch/geospatial/ip2geo/common/URLDenyListChecker.java

@@ -0,0 +1,55 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.geospatial.ip2geo.common;
+
+import inet.ipaddr.IPAddressString;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.geospatial.annotation.VisibleForTesting;
+
+import java.net.InetAddress;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.UnknownHostException;
+import java.util.List;
+
+/**
+ * A class to check url against a deny-list
+ */
+public class URLDenyListChecker {
+	private final ClusterSettings clusterSettings;
+	public URLDenyListChecker(final ClusterSettings clusterSettings) {
+		this.clusterSettings = clusterSettings;
+	}
+
+	/**
+	 * Convert String to URL after verifying the url is not on a deny-list
+	 *
+	 * @param url value to validate and convert to URL
+	 * @return value in URL type
+	 * @throws UnknownHostException
+	 * @throws MalformedURLException
+	 */
+	public URL toUrlIfNotInDenyList(final String url)
+		throws UnknownHostException, MalformedURLException {
+		return toUrlIfNotInDenyList(url, clusterSettings.get(Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST));
+	}
+
+	@VisibleForTesting
+	protected URL toUrlIfNotInDenyList(final String url, final List<String> denyList)
+		throws UnknownHostException, MalformedURLException {
+		URL urlToReturn = new URL(url);
+		if (isInDenyList(new IPAddressString(InetAddress.getByName(urlToReturn.getHost()).getHostAddress()), denyList)) {
+			throw new IllegalArgumentException("Given endpoint is blocked by deny list in cluster setting " + Ip2GeoSettings.DATASOURCE_ENDPOINT_DENYLIST.getKey());
+		}
+		return urlToReturn;
+	}
+
+	private boolean isInDenyList(final IPAddressString url, final List<String> denyList) {
+		return denyList.stream()
+			.map(cidr -> new IPAddressString(cidr))
+			.anyMatch(cidr -> cidr.contains(url));
+	}
+}

src/main/java/org/opensearch/geospatial/ip2geo/dao/GeoIpDataDao.java

@@ -60,6 +60,7 @@
 import org.opensearch.geospatial.annotation.VisibleForTesting;
 import org.opensearch.geospatial.constants.IndexSetting;
 import org.opensearch.geospatial.ip2geo.common.DatasourceManifest;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
 import org.opensearch.geospatial.shared.Constants;
 import org.opensearch.geospatial.shared.StashedThreadContext;
@@ -91,11 +92,13 @@ public class GeoIpDataDao {
     private final ClusterService clusterService;
     private final ClusterSettings clusterSettings;
     private final Client client;
+    private final URLDenyListChecker urlDenyListChecker;
 
-    public GeoIpDataDao(final ClusterService clusterService, final Client client) {
+    public GeoIpDataDao(final ClusterService clusterService, final Client client, final URLDenyListChecker urlDenyListChecker) {
         this.clusterService = clusterService;
         this.clusterSettings = clusterService.getClusterSettings();
         this.client = client;
+        this.urlDenyListChecker = urlDenyListChecker;
     }
 
     /**
@@ -172,7 +175,7 @@ public CSVParser getDatabaseReader(final DatasourceManifest manifest) {
         SpecialPermission.check();
         return AccessController.doPrivileged((PrivilegedAction<CSVParser>) () -> {
             try {
-                URL zipUrl = new URL(manifest.getUrl());
+                URL zipUrl = urlDenyListChecker.toUrlIfNotInDenyList(manifest.getUrl());
                 return internalGetDatabaseReader(manifest, zipUrl.openConnection());
             } catch (IOException e) {
                 throw new OpenSearchException("failed to read geoip data from {}", manifest.getUrl(), e);

src/main/java/org/opensearch/geospatial/ip2geo/jobscheduler/DatasourceUpdateService.java

@@ -25,6 +25,7 @@
 import org.opensearch.geospatial.annotation.VisibleForTesting;
 import org.opensearch.geospatial.ip2geo.common.DatasourceManifest;
 import org.opensearch.geospatial.ip2geo.common.DatasourceState;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.ip2geo.dao.DatasourceDao;
 import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao;
 import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule;
@@ -37,16 +38,19 @@ public class DatasourceUpdateService {
     private final ClusterSettings clusterSettings;
     private final DatasourceDao datasourceDao;
     private final GeoIpDataDao geoIpDataDao;
+    private final URLDenyListChecker urlDenyListChecker;
 
     public DatasourceUpdateService(
         final ClusterService clusterService,
         final DatasourceDao datasourceDao,
-        final GeoIpDataDao geoIpDataDao
+        final GeoIpDataDao geoIpDataDao,
+        final URLDenyListChecker urlDenyListChecker
     ) {
         this.clusterService = clusterService;
         this.clusterSettings = clusterService.getClusterSettings();
         this.datasourceDao = datasourceDao;
         this.geoIpDataDao = geoIpDataDao;
+        this.urlDenyListChecker = urlDenyListChecker;
     }
 
     /**
@@ -61,7 +65,7 @@ public DatasourceUpdateService(
      * @throws IOException
      */
     public void updateOrCreateGeoIpData(final Datasource datasource, final Runnable renewLock) throws IOException {
-        URL url = new URL(datasource.getEndpoint());
+        URL url = urlDenyListChecker.toUrlIfNotInDenyList(datasource.getEndpoint());
         DatasourceManifest manifest = DatasourceManifest.Builder.build(url);
 
         if (shouldUpdate(datasource, manifest) == false) {

src/main/java/org/opensearch/geospatial/plugin/GeospatialPlugin.java

@@ -54,6 +54,7 @@
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.ip2geo.dao.DatasourceDao;
 import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao;
 import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao;
@@ -97,6 +98,7 @@ public class GeospatialPlugin extends Plugin implements IngestPlugin, ActionPlug
     private Ip2GeoCachedDao ip2GeoCachedDao;
     private DatasourceDao datasourceDao;
     private GeoIpDataDao geoIpDataDao;
+    private URLDenyListChecker urlDenyListChecker;
 
     @Override
     public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
@@ -105,8 +107,9 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
 
     @Override
     public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
+        this.urlDenyListChecker = new URLDenyListChecker(parameters.ingestService.getClusterService().getClusterSettings());
         this.datasourceDao = new DatasourceDao(parameters.client, parameters.ingestService.getClusterService());
-        this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client);
+        this.geoIpDataDao = new GeoIpDataDao(parameters.ingestService.getClusterService(), parameters.client, urlDenyListChecker);
         this.ip2GeoCachedDao = new Ip2GeoCachedDao(parameters.ingestService.getClusterService(), datasourceDao, geoIpDataDao);
         return MapBuilder.<String, Processor.Factory>newMapBuilder()
             .put(FeatureProcessor.TYPE, new FeatureProcessor.Factory())
@@ -153,7 +156,7 @@ public Collection<Object> createComponents(
         IndexNameExpressionResolver indexNameExpressionResolver,
         Supplier<RepositoriesService> repositoriesServiceSupplier
     ) {
-        DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao);
+        DatasourceUpdateService datasourceUpdateService = new DatasourceUpdateService(clusterService, datasourceDao, geoIpDataDao, urlDenyListChecker);
         Ip2GeoExecutor ip2GeoExecutor = new Ip2GeoExecutor(threadPool);
         Ip2GeoLockService ip2GeoLockService = new Ip2GeoLockService(clusterService, client);
         /**
@@ -187,9 +190,9 @@ public List<RestHandler> getRestHandlers(
         List<RestHandler> geoJsonHandlers = List.of(new RestUploadStatsAction(), new RestUploadGeoJSONAction());
 
         List<RestHandler> ip2geoHandlers = List.of(
-            new RestPutDatasourceHandler(clusterSettings),
+            new RestPutDatasourceHandler(clusterSettings, urlDenyListChecker),
             new RestGetDatasourceHandler(),
-            new RestUpdateDatasourceHandler(),
+            new RestUpdateDatasourceHandler(urlDenyListChecker),
             new RestDeleteDatasourceHandler()
         );
 

src/test/java/org/opensearch/geospatial/ip2geo/Ip2GeoTestCase.java

@@ -47,6 +47,7 @@
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoExecutor;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoLockService;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.geospatial.ip2geo.dao.DatasourceDao;
 import org.opensearch.geospatial.ip2geo.dao.GeoIpDataDao;
 import org.opensearch.geospatial.ip2geo.dao.Ip2GeoCachedDao;
@@ -98,6 +99,7 @@ public abstract class Ip2GeoTestCase extends RestActionTestCase {
     protected Ip2GeoProcessorDao ip2GeoProcessorDao;
     @Mock
     protected RoutingTable routingTable;
+    protected URLDenyListChecker urlDenyListChecker;
     protected IngestMetadata ingestMetadata;
     protected NoOpNodeClient client;
     protected VerifyingClient verifyingClient;
@@ -115,6 +117,7 @@ public void prepareIp2GeoTestCase() {
         clusterSettings = new ClusterSettings(settings, new HashSet<>(Ip2GeoSettings.settings()));
         lockService = new LockService(client, clusterService);
         ingestMetadata = new IngestMetadata(Collections.emptyMap());
+        urlDenyListChecker = spy(new URLDenyListChecker(clusterSettings));
         when(metadata.custom(IngestMetadata.TYPE)).thenReturn(ingestMetadata);
         when(clusterService.getSettings()).thenReturn(Settings.EMPTY);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);

src/test/java/org/opensearch/geospatial/ip2geo/action/RestPutDatasourceHandlerTests.java

@@ -5,13 +5,16 @@
 
 package org.opensearch.geospatial.ip2geo.action;
 
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
 import static org.opensearch.geospatial.shared.URLBuilder.URL_DELIMITER;
 import static org.opensearch.geospatial.shared.URLBuilder.getPluginURLPrefix;
 
 import java.util.HashSet;
 import java.util.Locale;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import lombok.SneakyThrows;
 import org.junit.Before;
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.settings.ClusterSettings;
@@ -21,6 +24,7 @@
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.geospatial.GeospatialTestHelper;
 import org.opensearch.geospatial.ip2geo.common.Ip2GeoSettings;
+import org.opensearch.geospatial.ip2geo.common.URLDenyListChecker;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.test.rest.FakeRestRequest;
 import org.opensearch.test.rest.RestActionTestCase;
@@ -29,17 +33,22 @@
 public class RestPutDatasourceHandlerTests extends RestActionTestCase {
     private String path;
     private RestPutDatasourceHandler action;
+    private URLDenyListChecker urlDenyListChecker;
 
     @Before
     public void setupAction() {
-        action = new RestPutDatasourceHandler(new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings())));
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, new HashSet(Ip2GeoSettings.settings()));
+        urlDenyListChecker = mock(URLDenyListChecker.class);
+        action = new RestPutDatasourceHandler(clusterSettings, urlDenyListChecker);
         controller().registerHandler(action);
         path = String.join(URL_DELIMITER, getPluginURLPrefix(), "ip2geo/datasource/%s");
     }
 
+    @SneakyThrows
     public void testPrepareRequest() {
+        String endpoint = "https://test.com";
         String datasourceName = GeospatialTestHelper.randomLowerCaseString();
-        String content = "{\"endpoint\":\"https://test.com\", \"update_interval_in_days\":1}";
+        String content = String.format(Locale.ROOT, "{\"endpoint\":\"%s\", \"update_interval_in_days\":1}", endpoint);
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT)
             .withPath(String.format(Locale.ROOT, path, datasourceName))
             .withContent(new BytesArray(content), XContentType.JSON)
@@ -49,7 +58,7 @@ public void testPrepareRequest() {
         verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> {
             assertTrue(actionRequest instanceof PutDatasourceRequest);
             PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest;
-            assertEquals("https://test.com", putDatasourceRequest.getEndpoint());
+            assertEquals(endpoint, putDatasourceRequest.getEndpoint());
             assertEquals(TimeValue.timeValueDays(1), putDatasourceRequest.getUpdateInterval());
             assertEquals(datasourceName, putDatasourceRequest.getName());
             isExecuted.set(true);
@@ -58,9 +67,12 @@ public void testPrepareRequest() {
 
         dispatchRequest(request);
         assertTrue(isExecuted.get());
+        verify(urlDenyListChecker).toUrlIfNotInDenyList(endpoint);
     }
 
+    @SneakyThrows
     public void testPrepareRequestDefaultValue() {
+        String endpoint = "https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json";
         String datasourceName = GeospatialTestHelper.randomLowerCaseString();
         RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.PUT)
             .withPath(String.format(Locale.ROOT, path, datasourceName))
@@ -70,7 +82,7 @@ public void testPrepareRequestDefaultValue() {
         verifyingClient.setExecuteLocallyVerifier((actionResponse, actionRequest) -> {
             assertTrue(actionRequest instanceof PutDatasourceRequest);
             PutDatasourceRequest putDatasourceRequest = (PutDatasourceRequest) actionRequest;
-            assertEquals("https://geoip.maps.opensearch.org/v1/geolite2-city/manifest.json", putDatasourceRequest.getEndpoint());
+            assertEquals(endpoint, putDatasourceRequest.getEndpoint());
             assertEquals(TimeValue.timeValueDays(3), putDatasourceRequest.getUpdateInterval());
             assertEquals(datasourceName, putDatasourceRequest.getName());
             isExecuted.set(true);
@@ -79,5 +91,6 @@ public void testPrepareRequestDefaultValue() {
 
         dispatchRequest(request);
         assertTrue(isExecuted.get());
+        verify(urlDenyListChecker).toUrlIfNotInDenyList(endpoint);
     }
 }