diff --git a/src/main/java/nl/sidnlabs/entrada/enrich/resolver/OpenDNSResolverCheck.java b/src/main/java/nl/sidnlabs/entrada/enrich/resolver/OpenDNSResolverCheck.java index a84200a..d8e6a76 100644 --- a/src/main/java/nl/sidnlabs/entrada/enrich/resolver/OpenDNSResolverCheck.java +++ b/src/main/java/nl/sidnlabs/entrada/enrich/resolver/OpenDNSResolverCheck.java @@ -21,15 +21,19 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; +import org.apache.http.HttpEntity; import org.apache.http.client.config.RequestConfig; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.util.EntityUtils; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.select.Elements; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; -import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Setter; import lombok.extern.log4j.Log4j2; /** @@ -41,14 +45,13 @@ */ @Log4j2 @Component +@Setter public final class OpenDNSResolverCheck extends AbstractResolverCheck { private static final String RESOLVER_STATE_FILENAME = "opendns-resolvers"; private static final String RESOLVER_NAME = "OpenDNS"; - private static final ObjectMapper objectMapper = new ObjectMapper(); - @Value("${opendns.resolver.url}") private String url; @Value("${opendns.resolver.timeout:15}") @@ -71,26 +74,37 @@ protected List fetch() { try { HttpGet get = new HttpGet(url); CloseableHttpResponse response = client.execute(get); + HttpEntity entity = response.getEntity(); + String html = EntityUtils.toString(entity); + + Document doc = Jsoup.parse(html); + Elements tableRows = doc.select("table#networks tr"); + + if (tableRows.size() > 0) { + + // skip table header row + tableRows.stream().skip(1).forEach(r -> { + Elements columns = r.getElementsByTag("td"); + + String v4 = columns.get(3).text(); + String v6 = columns.get(4).text(); - @SuppressWarnings("unchecked") - List> locations = - objectMapper.readValue(response.getEntity().getContent(), List.class); + if (log.isDebugEnabled()) { + log + .debug("Add OpenDNS resolver IP ranges -> location: " + columns.get(0).text() + + " subnetV4: " + v4 + " subnetV6: " + v6); + } - for (Map location : locations) { - String v4 = location.get("subnetV4"); - String v6 = location.get("subnetV6"); + subnets.add(v4); + subnets.add(v6); - if (log.isDebugEnabled()) { - log.debug("Add OpenDNS resolver IP ranges, subnetV4: " + v4 + " subnetV6: " + v6); - } + }); - subnets.add(v4); - subnets.add(v6); } } catch (Exception e) { // ignore any problems when fetching resolver list - log.error("Problem while adding OpenDns resolvers."); + log.error("Problem while adding OpenDns resolvers.", e); } return subnets; } diff --git a/src/main/resources/application-dev.properties b/src/main/resources/application-dev.properties index 2757d2a..0160029 100644 --- a/src/main/resources/application-dev.properties +++ b/src/main/resources/application-dev.properties @@ -222,7 +222,7 @@ google.resolver.hostname=locations.publicdns.goog. google.resolver.timeout=15 # where to retrieve subnets of OpenDNS resolver -opendns.resolver.url=https://www.opendns.com/network-map-data +opendns.resolver.url=https://umbrella.cisco.com/why-umbrella/global-network-and-traffic # timeout in secs opendns.resolver.timeout=15 diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index db8f3a9..84d0f1f 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -220,7 +220,7 @@ google.resolver.hostname=locations.publicdns.goog. google.resolver.timeout=15 # where to retrieve subnets of OpenDNS resolver -opendns.resolver.url=https://www.opendns.com/network-map-data +opendns.resolver.url=https://umbrella.cisco.com/why-umbrella/global-network-and-traffic # timeout in secs opendns.resolver.timeout=15 diff --git a/src/test/java/nl/sidnlabs/entrada/enrich/resolver/OpenDnsResolverCheckTest.java b/src/test/java/nl/sidnlabs/entrada/enrich/resolver/OpenDnsResolverCheckTest.java new file mode 100644 index 0000000..4c9a6ff --- /dev/null +++ b/src/test/java/nl/sidnlabs/entrada/enrich/resolver/OpenDnsResolverCheckTest.java @@ -0,0 +1,20 @@ +package nl.sidnlabs.entrada.enrich.resolver; + +import static org.junit.Assert.assertTrue; +import java.util.List; +import org.junit.Test; + +public class OpenDnsResolverCheckTest { + + @Test + public void testLoadResolverEndpointsOk() { + + OpenDNSResolverCheck check = new OpenDNSResolverCheck(); + check.setUrl("https://umbrella.cisco.com/why-umbrella/global-network-and-traffic"); + check.setTimeout(5); + + List ips = check.fetch(); + assertTrue("No IPs found", ips.size() > 0); + } + +}