From a110d710db028f519a77d8bb73aa33e4365f1168 Mon Sep 17 00:00:00 2001 From: Ry Biesemeyer Date: Sun, 6 Oct 2024 10:26:30 -0700 Subject: [PATCH] ingest-geoip: establish boundaries (#109655) --- .../ingest/geoip/DatabaseNodeServiceIT.java | 2 +- ...gDatabasesWhilePerformingGeoLookupsIT.java | 4 +- .../elasticsearch/ingest/geoip/Database.java | 60 +- .../geoip/DatabaseReaderLazyLoader.java | 130 +--- .../ingest/geoip/GeoIpCache.java | 2 +- .../ingest/geoip/GeoIpProcessor.java | 535 +--------------- .../ingest/geoip/IpDataLookup.java | 31 + .../ingest/geoip/IpDataLookupFactories.java | 107 ++++ .../ingest/geoip/IpDatabase.java | 53 +- .../ingest/geoip/MaxmindIpDataLookups.java | 606 ++++++++++++++++++ .../ingest/geoip/ConfigDatabasesTests.java | 4 +- .../ingest/geoip/GeoIpProcessorTests.java | 58 +- .../ingest/geoip/GeoIpTestUtils.java | 32 + .../ingest/geoip/MMDBUtilTests.java | 2 +- .../ingest/geoip/MaxMindSupportTests.java | 49 -- 15 files changed, 840 insertions(+), 835 deletions(-) create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java create mode 100644 modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java index 73d8976c3a4b7..786f091e0c024 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/DatabaseNodeServiceIT.java @@ -84,7 +84,7 @@ private void assertValidDatabase(DatabaseNodeService databaseNodeService, String IpDatabase database = databaseNodeService.getDatabase(databaseFileName); assertNotNull(database); assertThat(database.getDatabaseType(), equalTo(databaseType)); - CountryResponse countryResponse = database.getCountry("89.160.20.128"); + CountryResponse countryResponse = database.getResponse("89.160.20.128", GeoIpTestUtils::getCountry); assertNotNull(countryResponse); Country country = countryResponse.getCountry(); assertNotNull(country); diff --git a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java index 2c7d5fbcc56b7..b28926673069d 100644 --- a/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java +++ b/modules/ingest-geoip/src/internalClusterTest/java/org/elasticsearch/ingest/geoip/ReloadingDatabasesWhilePerformingGeoLookupsIT.java @@ -205,10 +205,10 @@ private static DatabaseNodeService createRegistry(Path geoIpConfigDir, Path geoI private static void lazyLoadReaders(DatabaseNodeService databaseNodeService) throws IOException { if (databaseNodeService.get("GeoLite2-City.mmdb") != null) { databaseNodeService.get("GeoLite2-City.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City.mmdb").getCity("2.125.160.216"); + databaseNodeService.get("GeoLite2-City.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } databaseNodeService.get("GeoLite2-City-Test.mmdb").getDatabaseType(); - databaseNodeService.get("GeoLite2-City-Test.mmdb").getCity("2.125.160.216"); + databaseNodeService.get("GeoLite2-City-Test.mmdb").getResponse("2.125.160.216", GeoIpTestUtils::getCity); } } diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java index dccda0d58cfbf..52ca5eea52c1a 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java @@ -9,7 +9,6 @@ package org.elasticsearch.ingest.geoip; -import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import java.util.Arrays; @@ -19,12 +18,10 @@ import java.util.Set; /** - * A high-level representation of a kind of geoip database that is supported by the {@link GeoIpProcessor}. + * A high-level representation of a kind of ip location database that is supported by the {@link GeoIpProcessor}. *

* A database has a set of properties that are valid to use with it (see {@link Database#properties()}), * as well as a list of default properties to use if no properties are specified (see {@link Database#defaultProperties()}). - *

- * See especially {@link Database#getDatabase(String, String)} which is used to obtain instances of this class. */ enum Database { @@ -142,61 +139,6 @@ enum Database { ) ); - private static final String CITY_DB_SUFFIX = "-City"; - private static final String COUNTRY_DB_SUFFIX = "-Country"; - private static final String ASN_DB_SUFFIX = "-ASN"; - private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; - private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; - private static final String DOMAIN_DB_SUFFIX = "-Domain"; - private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; - private static final String ISP_DB_SUFFIX = "-ISP"; - - @Nullable - private static Database getMaxmindDatabase(final String databaseType) { - if (databaseType.endsWith(Database.CITY_DB_SUFFIX)) { - return Database.City; - } else if (databaseType.endsWith(Database.COUNTRY_DB_SUFFIX)) { - return Database.Country; - } else if (databaseType.endsWith(Database.ASN_DB_SUFFIX)) { - return Database.Asn; - } else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) { - return Database.AnonymousIp; - } else if (databaseType.endsWith(Database.CONNECTION_TYPE_DB_SUFFIX)) { - return Database.ConnectionType; - } else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) { - return Database.Domain; - } else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) { - return Database.Enterprise; - } else if (databaseType.endsWith(Database.ISP_DB_SUFFIX)) { - return Database.Isp; - } else { - return null; // no match was found - } - } - - /** - * Parses the passed-in databaseType (presumably from the passed-in databaseFile) and return the Database instance that is - * associated with that databaseType. - * - * @param databaseType the database type String from the metadata of the database file - * @param databaseFile the database file from which the database type was obtained - * @throws IllegalArgumentException if the databaseType is not associated with a Database instance - * @return the Database instance that is associated with the databaseType - */ - public static Database getDatabase(final String databaseType, final String databaseFile) { - Database database = null; - - if (Strings.hasText(databaseType)) { - database = getMaxmindDatabase(databaseType); - } - - if (database == null) { - throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); - } - - return database; - } - private final Set properties; private final Set defaultProperties; diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java index e160c8ad1543f..120afe0e9e815 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java @@ -9,18 +9,8 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.DatabaseRecord; -import com.maxmind.db.Network; import com.maxmind.db.NoCache; import com.maxmind.db.Reader; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -28,8 +18,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.CheckedSupplier; -import org.elasticsearch.common.network.InetAddresses; -import org.elasticsearch.common.network.NetworkAddress; import org.elasticsearch.core.Booleans; import org.elasticsearch.core.IOUtils; import org.elasticsearch.core.Nullable; @@ -37,19 +25,16 @@ import java.io.File; import java.io.IOException; -import java.net.InetAddress; import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; /** * Facilitates lazy loading of the database reader, so that when the geoip plugin is installed, but not used, * no memory is being wasted on the database reader. */ -class DatabaseReaderLazyLoader implements IpDatabase { +public class DatabaseReaderLazyLoader implements IpDatabase { private static final boolean LOAD_DATABASE_ON_HEAP = Booleans.parseBoolean(System.getProperty("es.geoip.load_db_on_heap", "false")); @@ -96,94 +81,6 @@ public final String getDatabaseType() throws IOException { return databaseType.get(); } - @Nullable - @Override - public CityResponse getCity(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, CityResponse.class, CityResponse::new)); - } - - @Nullable - @Override - public CountryResponse getCountry(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, CountryResponse.class, CountryResponse::new)); - } - - @Nullable - @Override - public AsnResponse getAsn(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - AsnResponse.class, - (response, responseIp, network, locales) -> new AsnResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public AnonymousIpResponse getAnonymousIp(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - AnonymousIpResponse.class, - (response, responseIp, network, locales) -> new AnonymousIpResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public ConnectionTypeResponse getConnectionType(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - ConnectionTypeResponse.class, - (response, responseIp, network, locales) -> new ConnectionTypeResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public DomainResponse getDomain(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - DomainResponse.class, - (response, responseIp, network, locales) -> new DomainResponse(response, responseIp, network) - ) - ); - } - - @Nullable - @Override - public EnterpriseResponse getEnterprise(String ipAddress) { - return getResponse(ipAddress, (reader, ip) -> lookup(reader, ip, EnterpriseResponse.class, EnterpriseResponse::new)); - } - - @Nullable - @Override - public IspResponse getIsp(String ipAddress) { - return getResponse( - ipAddress, - (reader, ip) -> lookup( - reader, - ip, - IspResponse.class, - (response, responseIp, network, locales) -> new IspResponse(response, responseIp, network) - ) - ); - } - boolean preLookup() { return currentUsages.updateAndGet(current -> current < 0 ? current : current + 1) > 0; } @@ -199,14 +96,12 @@ int current() { return currentUsages.get(); } + @Override @Nullable - private RESPONSE getResponse( - String ipAddress, - CheckedBiFunction, Exception> responseProvider - ) { + public RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider) { return cache.putIfAbsent(ipAddress, databasePath.toString(), ip -> { try { - return responseProvider.apply(get(), ipAddress).orElse(null); + return responseProvider.apply(get(), ipAddress); } catch (Exception e) { throw ExceptionsHelper.convertToRuntime(e); } @@ -263,23 +158,6 @@ private static File pathToFile(Path databasePath) { return databasePath.toFile(); } - @FunctionalInterface - private interface ResponseBuilder { - RESPONSE build(RESPONSE response, String responseIp, Network network, List locales); - } - - private Optional lookup(Reader reader, String ip, Class clazz, ResponseBuilder builder) - throws IOException { - InetAddress inetAddress = InetAddresses.forString(ip); - DatabaseRecord record = reader.getRecord(inetAddress, clazz); - RESPONSE result = record.getData(); - if (result == null) { - return Optional.empty(); - } else { - return Optional.of(builder.build(result, NetworkAddress.format(inetAddress), record.getNetwork(), List.of("en"))); - } - } - long getBuildDateMillis() throws IOException { if (buildDate.get() == null) { synchronized (buildDate) { diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java index 335331ac0ab9d..d9c9c3aaf3266 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpCache.java @@ -26,7 +26,7 @@ * cost of deserialization for each lookup (cached or not). This comes at slight expense of higher memory usage, but significant * reduction of CPU usage. */ -final class GeoIpCache { +public final class GeoIpCache { /** * Internal-only sentinel object for recording that a result from the geoip database was null (i.e. there was no result). By caching diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java index ce160b060ae4c..e2b516bf5b943 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java @@ -9,23 +9,6 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.db.Network; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse.ConnectionType; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; -import com.maxmind.geoip2.record.City; -import com.maxmind.geoip2.record.Continent; -import com.maxmind.geoip2.record.Country; -import com.maxmind.geoip2.record.Location; -import com.maxmind.geoip2.record.Subdivision; - -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; @@ -34,10 +17,10 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.geoip.Database.Property; +import org.elasticsearch.ingest.geoip.IpDataLookupFactories.IpDataLookupFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -60,7 +43,7 @@ public final class GeoIpProcessor extends AbstractProcessor { private final Supplier isValid; private final String targetField; private final CheckedSupplier supplier; - private final Set properties; + private final IpDataLookup ipDataLookup; private final boolean ignoreMissing; private final boolean firstOnly; private final String databaseFile; @@ -73,7 +56,7 @@ public final class GeoIpProcessor extends AbstractProcessor { * @param supplier a supplier of a geo-IP database reader; ideally this is lazily-loaded once on first use * @param isValid a supplier that determines if the available database files are up-to-date and license compliant * @param targetField the target field - * @param properties the properties; ideally this is lazily-loaded once on first use + * @param ipDataLookup a lookup capable of retrieving a result from an available geo-IP database reader * @param ignoreMissing true if documents with a missing value for the field should be ignored * @param firstOnly true if only first result should be returned in case of array * @param databaseFile the name of the database file being queried; used only for tagging documents if the database is unavailable @@ -85,7 +68,7 @@ public final class GeoIpProcessor extends AbstractProcessor { final CheckedSupplier supplier, final Supplier isValid, final String targetField, - final Set properties, + final IpDataLookup ipDataLookup, final boolean ignoreMissing, final boolean firstOnly, final String databaseFile @@ -95,7 +78,7 @@ public final class GeoIpProcessor extends AbstractProcessor { this.isValid = isValid; this.targetField = targetField; this.supplier = supplier; - this.properties = properties; + this.ipDataLookup = ipDataLookup; this.ignoreMissing = ignoreMissing; this.firstOnly = firstOnly; this.databaseFile = databaseFile; @@ -127,7 +110,7 @@ public IngestDocument execute(IngestDocument document) throws IOException { } if (ip instanceof String ipString) { - Map data = getGeoData(ipDatabase, ipString); + Map data = ipDataLookup.getData(ipDatabase, ipString); if (data.isEmpty() == false) { document.setFieldValue(targetField, data); } @@ -138,7 +121,7 @@ public IngestDocument execute(IngestDocument document) throws IOException { if (ipAddr instanceof String == false) { throw new IllegalArgumentException("array in field [" + field + "] should only contain strings"); } - Map data = getGeoData(ipDatabase, (String) ipAddr); + Map data = ipDataLookup.getData(ipDatabase, (String) ipAddr); if (data.isEmpty()) { dataList.add(null); continue; @@ -161,26 +144,6 @@ public IngestDocument execute(IngestDocument document) throws IOException { return document; } - private Map getGeoData(IpDatabase ipDatabase, String ipAddress) throws IOException { - final String databaseType = ipDatabase.getDatabaseType(); - final Database database; - try { - database = Database.getDatabase(databaseType, databaseFile); - } catch (IllegalArgumentException e) { - throw new ElasticsearchParseException(e.getMessage(), e); - } - return switch (database) { - case City -> retrieveCityGeoData(ipDatabase, ipAddress); - case Country -> retrieveCountryGeoData(ipDatabase, ipAddress); - case Asn -> retrieveAsnGeoData(ipDatabase, ipAddress); - case AnonymousIp -> retrieveAnonymousIpGeoData(ipDatabase, ipAddress); - case ConnectionType -> retrieveConnectionTypeGeoData(ipDatabase, ipAddress); - case Domain -> retrieveDomainGeoData(ipDatabase, ipAddress); - case Enterprise -> retrieveEnterpriseGeoData(ipDatabase, ipAddress); - case Isp -> retrieveIspGeoData(ipDatabase, ipAddress); - }; - } - @Override public String getType() { return TYPE; @@ -199,478 +162,7 @@ String getDatabaseType() throws IOException { } Set getProperties() { - return properties; - } - - private Map retrieveCityGeoData(IpDatabase ipDatabase, String ipAddress) { - CityResponse response = ipDatabase.getCity(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - } - } - return geoData; - } - - private Map retrieveCountryGeoData(IpDatabase ipDatabase, String ipAddress) { - CountryResponse response = ipDatabase.getCountry(ipAddress); - if (response == null) { - return Map.of(); - } - Country country = response.getCountry(); - Continent continent = response.getContinent(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - } - } - return geoData; - } - - private Map retrieveAsnGeoData(IpDatabase ipDatabase, String ipAddress) { - AsnResponse response = ipDatabase.getAsn(ipAddress); - if (response == null) { - return Map.of(); - } - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveAnonymousIpGeoData(IpDatabase ipDatabase, String ipAddress) { - AnonymousIpResponse response = ipDatabase.getAnonymousIp(ipAddress); - if (response == null) { - return Map.of(); - } - - boolean isHostingProvider = response.isHostingProvider(); - boolean isTorExitNode = response.isTorExitNode(); - boolean isAnonymousVpn = response.isAnonymousVpn(); - boolean isAnonymous = response.isAnonymous(); - boolean isPublicProxy = response.isPublicProxy(); - boolean isResidentialProxy = response.isResidentialProxy(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - } - } - return geoData; - } - - private Map retrieveConnectionTypeGeoData(IpDatabase ipDatabase, String ipAddress) { - ConnectionTypeResponse response = ipDatabase.getConnectionType(ipAddress); - if (response == null) { - return Map.of(); - } - - ConnectionType connectionType = response.getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveDomainGeoData(IpDatabase ipDatabase, String ipAddress) { - DomainResponse response = ipDatabase.getDomain(ipAddress); - if (response == null) { - return Map.of(); - } - - String domain = response.getDomain(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - } - } - return geoData; - } - - private Map retrieveEnterpriseGeoData(IpDatabase ipDatabase, String ipAddress) { - EnterpriseResponse response = ipDatabase.getEnterprise(ipAddress); - if (response == null) { - return Map.of(); - } - - Country country = response.getCountry(); - City city = response.getCity(); - Location location = response.getLocation(); - Continent continent = response.getContinent(); - Subdivision subdivision = response.getMostSpecificSubdivision(); - - Long asn = response.getTraits().getAutonomousSystemNumber(); - String organizationName = response.getTraits().getAutonomousSystemOrganization(); - Network network = response.getTraits().getNetwork(); - - String isp = response.getTraits().getIsp(); - String ispOrganization = response.getTraits().getOrganization(); - String mobileCountryCode = response.getTraits().getMobileCountryCode(); - String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); - - boolean isHostingProvider = response.getTraits().isHostingProvider(); - boolean isTorExitNode = response.getTraits().isTorExitNode(); - boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); - boolean isAnonymous = response.getTraits().isAnonymous(); - boolean isPublicProxy = response.getTraits().isPublicProxy(); - boolean isResidentialProxy = response.getTraits().isResidentialProxy(); - - String userType = response.getTraits().getUserType(); - - String domain = response.getTraits().getDomain(); - - ConnectionType connectionType = response.getTraits().getConnectionType(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getTraits().getIpAddress()); - case COUNTRY_ISO_CODE -> { - String countryIsoCode = country.getIsoCode(); - if (countryIsoCode != null) { - geoData.put("country_iso_code", countryIsoCode); - } - } - case COUNTRY_NAME -> { - String countryName = country.getName(); - if (countryName != null) { - geoData.put("country_name", countryName); - } - } - case CONTINENT_CODE -> { - String continentCode = continent.getCode(); - if (continentCode != null) { - geoData.put("continent_code", continentCode); - } - } - case CONTINENT_NAME -> { - String continentName = continent.getName(); - if (continentName != null) { - geoData.put("continent_name", continentName); - } - } - case REGION_ISO_CODE -> { - // ISO 3166-2 code for country subdivisions. - // See iso.org/iso-3166-country-codes.html - String countryIso = country.getIsoCode(); - String subdivisionIso = subdivision.getIsoCode(); - if (countryIso != null && subdivisionIso != null) { - String regionIsoCode = countryIso + "-" + subdivisionIso; - geoData.put("region_iso_code", regionIsoCode); - } - } - case REGION_NAME -> { - String subdivisionName = subdivision.getName(); - if (subdivisionName != null) { - geoData.put("region_name", subdivisionName); - } - } - case CITY_NAME -> { - String cityName = city.getName(); - if (cityName != null) { - geoData.put("city_name", cityName); - } - } - case TIMEZONE -> { - String locationTimeZone = location.getTimeZone(); - if (locationTimeZone != null) { - geoData.put("timezone", locationTimeZone); - } - } - case LOCATION -> { - Double latitude = location.getLatitude(); - Double longitude = location.getLongitude(); - if (latitude != null && longitude != null) { - Map locationObject = new HashMap<>(); - locationObject.put("lat", latitude); - locationObject.put("lon", longitude); - geoData.put("location", locationObject); - } - } - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case HOSTING_PROVIDER -> { - geoData.put("hosting_provider", isHostingProvider); - } - case TOR_EXIT_NODE -> { - geoData.put("tor_exit_node", isTorExitNode); - } - case ANONYMOUS_VPN -> { - geoData.put("anonymous_vpn", isAnonymousVpn); - } - case ANONYMOUS -> { - geoData.put("anonymous", isAnonymous); - } - case PUBLIC_PROXY -> { - geoData.put("public_proxy", isPublicProxy); - } - case RESIDENTIAL_PROXY -> { - geoData.put("residential_proxy", isResidentialProxy); - } - case DOMAIN -> { - if (domain != null) { - geoData.put("domain", domain); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - case USER_TYPE -> { - if (userType != null) { - geoData.put("user_type", userType); - } - } - case CONNECTION_TYPE -> { - if (connectionType != null) { - geoData.put("connection_type", connectionType.toString()); - } - } - } - } - return geoData; - } - - private Map retrieveIspGeoData(IpDatabase ipDatabase, String ipAddress) { - IspResponse response = ipDatabase.getIsp(ipAddress); - if (response == null) { - return Map.of(); - } - - String isp = response.getIsp(); - String ispOrganization = response.getOrganization(); - String mobileNetworkCode = response.getMobileNetworkCode(); - String mobileCountryCode = response.getMobileCountryCode(); - Long asn = response.getAutonomousSystemNumber(); - String organizationName = response.getAutonomousSystemOrganization(); - Network network = response.getNetwork(); - - Map geoData = new HashMap<>(); - for (Property property : this.properties) { - switch (property) { - case IP -> geoData.put("ip", response.getIpAddress()); - case ASN -> { - if (asn != null) { - geoData.put("asn", asn); - } - } - case ORGANIZATION_NAME -> { - if (organizationName != null) { - geoData.put("organization_name", organizationName); - } - } - case NETWORK -> { - if (network != null) { - geoData.put("network", network.toString()); - } - } - case ISP -> { - if (isp != null) { - geoData.put("isp", isp); - } - } - case ISP_ORGANIZATION_NAME -> { - if (ispOrganization != null) { - geoData.put("isp_organization_name", ispOrganization); - } - } - case MOBILE_COUNTRY_CODE -> { - if (mobileCountryCode != null) { - geoData.put("mobile_country_code", mobileCountryCode); - } - } - case MOBILE_NETWORK_CODE -> { - if (mobileNetworkCode != null) { - geoData.put("mobile_network_code", mobileNetworkCode); - } - } - } - } - return geoData; + return ipDataLookup.getProperties(); } /** @@ -752,19 +244,20 @@ public Processor create( databaseType = ipDatabase.getDatabaseType(); } - final Database database; + final IpDataLookupFactory factory; try { - database = Database.getDatabase(databaseType, databaseFile); + factory = IpDataLookupFactories.get(databaseType, databaseFile); } catch (IllegalArgumentException e) { throw newConfigurationException(TYPE, processorTag, "database_file", e.getMessage()); } - final Set properties; + final IpDataLookup ipDataLookup; try { - properties = database.parseProperties(propertyNames); + ipDataLookup = factory.create(propertyNames); } catch (IllegalArgumentException e) { throw newConfigurationException(TYPE, processorTag, "properties", e.getMessage()); } + return new GeoIpProcessor( processorTag, description, @@ -772,7 +265,7 @@ public Processor create( new DatabaseVerifyingSupplier(ipDatabaseProvider, databaseFile, databaseType), () -> ipDatabaseProvider.isValid(databaseFile), targetField, - properties, + ipDataLookup, ignoreMissing, firstOnly, databaseFile diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java new file mode 100644 index 0000000000000..7442c8e930886 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookup.java @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import java.io.IOException; +import java.util.Map; +import java.util.Set; + +interface IpDataLookup { + /** + * Gets data from the provided {@code ipDatabase} for the provided {@code ip} + * + * @param ipDatabase the database from which to lookup a result + * @param ip the ip address + * @return a map of data corresponding to the configured properties + * @throws IOException if the implementation encounters any problem while retrieving the response + */ + Map getData(IpDatabase ipDatabase, String ip) throws IOException; + + /** + * @return the set of properties this lookup will provide + */ + Set getProperties(); +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java new file mode 100644 index 0000000000000..990788978a0ca --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDataLookupFactories.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.core.Nullable; + +import java.util.List; +import java.util.Set; +import java.util.function.Function; + +final class IpDataLookupFactories { + + private IpDataLookupFactories() { + // utility class + } + + interface IpDataLookupFactory { + IpDataLookup create(List properties); + } + + private static final String CITY_DB_SUFFIX = "-City"; + private static final String COUNTRY_DB_SUFFIX = "-Country"; + private static final String ASN_DB_SUFFIX = "-ASN"; + private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP"; + private static final String CONNECTION_TYPE_DB_SUFFIX = "-Connection-Type"; + private static final String DOMAIN_DB_SUFFIX = "-Domain"; + private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise"; + private static final String ISP_DB_SUFFIX = "-ISP"; + + @Nullable + private static Database getMaxmindDatabase(final String databaseType) { + if (databaseType.endsWith(CITY_DB_SUFFIX)) { + return Database.City; + } else if (databaseType.endsWith(COUNTRY_DB_SUFFIX)) { + return Database.Country; + } else if (databaseType.endsWith(ASN_DB_SUFFIX)) { + return Database.Asn; + } else if (databaseType.endsWith(ANONYMOUS_IP_DB_SUFFIX)) { + return Database.AnonymousIp; + } else if (databaseType.endsWith(CONNECTION_TYPE_DB_SUFFIX)) { + return Database.ConnectionType; + } else if (databaseType.endsWith(DOMAIN_DB_SUFFIX)) { + return Database.Domain; + } else if (databaseType.endsWith(ENTERPRISE_DB_SUFFIX)) { + return Database.Enterprise; + } else if (databaseType.endsWith(ISP_DB_SUFFIX)) { + return Database.Isp; + } else { + return null; // no match was found + } + } + + /** + * Parses the passed-in databaseType and return the Database instance that is + * associated with that databaseType. + * + * @param databaseType the database type String from the metadata of the database file + * @return the Database instance that is associated with the databaseType + */ + @Nullable + static Database getDatabase(final String databaseType) { + Database database = null; + + if (Strings.hasText(databaseType)) { + database = getMaxmindDatabase(databaseType); + } + + return database; + } + + static Function, IpDataLookup> getMaxmindLookup(final Database database) { + return switch (database) { + case City -> MaxmindIpDataLookups.City::new; + case Country -> MaxmindIpDataLookups.Country::new; + case Asn -> MaxmindIpDataLookups.Asn::new; + case AnonymousIp -> MaxmindIpDataLookups.AnonymousIp::new; + case ConnectionType -> MaxmindIpDataLookups.ConnectionType::new; + case Domain -> MaxmindIpDataLookups.Domain::new; + case Enterprise -> MaxmindIpDataLookups.Enterprise::new; + case Isp -> MaxmindIpDataLookups.Isp::new; + }; + } + + static IpDataLookupFactory get(final String databaseType, final String databaseFile) { + final Database database = getDatabase(databaseType); + if (database == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + final Function, IpDataLookup> factoryMethod = getMaxmindLookup(database); + + // note: this can't presently be null, but keep this check -- it will be useful in the near future + if (factoryMethod == null) { + throw new IllegalArgumentException("Unsupported database type [" + databaseType + "] for file [" + databaseFile + "]"); + } + + return (properties) -> factoryMethod.apply(database.parseProperties(properties)); + } +} diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java index f416259a87d27..db1ffc1c682b8 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IpDatabase.java @@ -9,15 +9,9 @@ package org.elasticsearch.ingest.geoip; -import com.maxmind.geoip2.model.AnonymousIpResponse; -import com.maxmind.geoip2.model.AsnResponse; -import com.maxmind.geoip2.model.CityResponse; -import com.maxmind.geoip2.model.ConnectionTypeResponse; -import com.maxmind.geoip2.model.CountryResponse; -import com.maxmind.geoip2.model.DomainResponse; -import com.maxmind.geoip2.model.EnterpriseResponse; -import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.db.Reader; +import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.core.Nullable; import java.io.IOException; @@ -34,44 +28,15 @@ public interface IpDatabase extends AutoCloseable { String getDatabaseType() throws IOException; /** - * @param ipAddress the IP address to look up - * @return a response containing the city data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving city data - */ - @Nullable - CityResponse getCity(String ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the country data for the given address if it exists, or null if it could not be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving country data - */ - @Nullable - CountryResponse getCountry(String ipAddress); - - /** - * @param ipAddress the IP address to look up - * @return a response containing the Autonomous System Number for the given address if it exists, or null if it could not - * be found - * @throws UnsupportedOperationException may be thrown if the implementation does not support retrieving ASN data + * Returns a response from this database's reader for the given IP address. + * + * @param ipAddress the address to lookup + * @param responseProvider a method for extracting a response from a {@link Reader}, usually this will be a method reference + * @return a possibly-null response + * @param the type of response that will be returned */ @Nullable - AsnResponse getAsn(String ipAddress); - - @Nullable - AnonymousIpResponse getAnonymousIp(String ipAddress); - - @Nullable - ConnectionTypeResponse getConnectionType(String ipAddress); - - @Nullable - DomainResponse getDomain(String ipAddress); - - @Nullable - EnterpriseResponse getEnterprise(String ipAddress); - - @Nullable - IspResponse getIsp(String ipAddress); + RESPONSE getResponse(String ipAddress, CheckedBiFunction responseProvider); /** * Releases the current database object. Called after processing a single document. Databases should be closed or returned to a diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java new file mode 100644 index 0000000000000..5b22b3f4005a9 --- /dev/null +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java @@ -0,0 +1,606 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.ingest.geoip; + +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Network; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.AbstractResponse; +import com.maxmind.geoip2.model.AnonymousIpResponse; +import com.maxmind.geoip2.model.AsnResponse; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.ConnectionTypeResponse; +import com.maxmind.geoip2.model.CountryResponse; +import com.maxmind.geoip2.model.DomainResponse; +import com.maxmind.geoip2.model.EnterpriseResponse; +import com.maxmind.geoip2.model.IspResponse; +import com.maxmind.geoip2.record.Continent; +import com.maxmind.geoip2.record.Location; +import com.maxmind.geoip2.record.Subdivision; + +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.core.Nullable; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * A collection of {@link IpDataLookup} implementations for MaxMind databases + */ +final class MaxmindIpDataLookups { + + private MaxmindIpDataLookups() { + // utility class + } + + static class AnonymousIp extends AbstractBase { + AnonymousIp(final Set properties) { + super( + properties, + AnonymousIpResponse.class, + (response, ipAddress, network, locales) -> new AnonymousIpResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final AnonymousIpResponse response) { + boolean isHostingProvider = response.isHostingProvider(); + boolean isTorExitNode = response.isTorExitNode(); + boolean isAnonymousVpn = response.isAnonymousVpn(); + boolean isAnonymous = response.isAnonymous(); + boolean isPublicProxy = response.isPublicProxy(); + boolean isResidentialProxy = response.isResidentialProxy(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + } + } + return data; + } + } + + static class Asn extends AbstractBase { + Asn(Set properties) { + super(properties, AsnResponse.class, (response, ipAddress, network, locales) -> new AsnResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final AsnResponse response) { + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + } + } + return data; + } + } + + static class City extends AbstractBase { + City(final Set properties) { + super(properties, CityResponse.class, CityResponse::new); + } + + @Override + protected Map transform(final CityResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + } + } + return data; + } + } + + static class ConnectionType extends AbstractBase { + ConnectionType(final Set properties) { + super( + properties, + ConnectionTypeResponse.class, + (response, ipAddress, network, locales) -> new ConnectionTypeResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final ConnectionTypeResponse response) { + ConnectionTypeResponse.ConnectionType connectionType = response.getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + } + } + return data; + } + } + + static class Country extends AbstractBase { + Country(final Set properties) { + super(properties, CountryResponse.class, CountryResponse::new); + } + + @Override + protected Map transform(final CountryResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + Continent continent = response.getContinent(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + } + } + return data; + } + } + + static class Domain extends AbstractBase { + Domain(final Set properties) { + super( + properties, + DomainResponse.class, + (response, ipAddress, network, locales) -> new DomainResponse(response, ipAddress, network) + ); + } + + @Override + protected Map transform(final DomainResponse response) { + String domain = response.getDomain(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + } + } + return data; + } + } + + static class Enterprise extends AbstractBase { + Enterprise(final Set properties) { + super(properties, EnterpriseResponse.class, EnterpriseResponse::new); + } + + @Override + protected Map transform(final EnterpriseResponse response) { + com.maxmind.geoip2.record.Country country = response.getCountry(); + com.maxmind.geoip2.record.City city = response.getCity(); + Location location = response.getLocation(); + Continent continent = response.getContinent(); + Subdivision subdivision = response.getMostSpecificSubdivision(); + + Long asn = response.getTraits().getAutonomousSystemNumber(); + String organizationName = response.getTraits().getAutonomousSystemOrganization(); + Network network = response.getTraits().getNetwork(); + + String isp = response.getTraits().getIsp(); + String ispOrganization = response.getTraits().getOrganization(); + String mobileCountryCode = response.getTraits().getMobileCountryCode(); + String mobileNetworkCode = response.getTraits().getMobileNetworkCode(); + + boolean isHostingProvider = response.getTraits().isHostingProvider(); + boolean isTorExitNode = response.getTraits().isTorExitNode(); + boolean isAnonymousVpn = response.getTraits().isAnonymousVpn(); + boolean isAnonymous = response.getTraits().isAnonymous(); + boolean isPublicProxy = response.getTraits().isPublicProxy(); + boolean isResidentialProxy = response.getTraits().isResidentialProxy(); + + String userType = response.getTraits().getUserType(); + + String domain = response.getTraits().getDomain(); + + ConnectionTypeResponse.ConnectionType connectionType = response.getTraits().getConnectionType(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getTraits().getIpAddress()); + case COUNTRY_ISO_CODE -> { + String countryIsoCode = country.getIsoCode(); + if (countryIsoCode != null) { + data.put("country_iso_code", countryIsoCode); + } + } + case COUNTRY_NAME -> { + String countryName = country.getName(); + if (countryName != null) { + data.put("country_name", countryName); + } + } + case CONTINENT_CODE -> { + String continentCode = continent.getCode(); + if (continentCode != null) { + data.put("continent_code", continentCode); + } + } + case CONTINENT_NAME -> { + String continentName = continent.getName(); + if (continentName != null) { + data.put("continent_name", continentName); + } + } + case REGION_ISO_CODE -> { + // ISO 3166-2 code for country subdivisions. + // See iso.org/iso-3166-country-codes.html + String countryIso = country.getIsoCode(); + String subdivisionIso = subdivision.getIsoCode(); + if (countryIso != null && subdivisionIso != null) { + String regionIsoCode = countryIso + "-" + subdivisionIso; + data.put("region_iso_code", regionIsoCode); + } + } + case REGION_NAME -> { + String subdivisionName = subdivision.getName(); + if (subdivisionName != null) { + data.put("region_name", subdivisionName); + } + } + case CITY_NAME -> { + String cityName = city.getName(); + if (cityName != null) { + data.put("city_name", cityName); + } + } + case TIMEZONE -> { + String locationTimeZone = location.getTimeZone(); + if (locationTimeZone != null) { + data.put("timezone", locationTimeZone); + } + } + case LOCATION -> { + Double latitude = location.getLatitude(); + Double longitude = location.getLongitude(); + if (latitude != null && longitude != null) { + Map locationObject = new HashMap<>(); + locationObject.put("lat", latitude); + locationObject.put("lon", longitude); + data.put("location", locationObject); + } + } + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case HOSTING_PROVIDER -> { + data.put("hosting_provider", isHostingProvider); + } + case TOR_EXIT_NODE -> { + data.put("tor_exit_node", isTorExitNode); + } + case ANONYMOUS_VPN -> { + data.put("anonymous_vpn", isAnonymousVpn); + } + case ANONYMOUS -> { + data.put("anonymous", isAnonymous); + } + case PUBLIC_PROXY -> { + data.put("public_proxy", isPublicProxy); + } + case RESIDENTIAL_PROXY -> { + data.put("residential_proxy", isResidentialProxy); + } + case DOMAIN -> { + if (domain != null) { + data.put("domain", domain); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + case USER_TYPE -> { + if (userType != null) { + data.put("user_type", userType); + } + } + case CONNECTION_TYPE -> { + if (connectionType != null) { + data.put("connection_type", connectionType.toString()); + } + } + } + } + return data; + } + } + + static class Isp extends AbstractBase { + Isp(final Set properties) { + super(properties, IspResponse.class, (response, ipAddress, network, locales) -> new IspResponse(response, ipAddress, network)); + } + + @Override + protected Map transform(final IspResponse response) { + String isp = response.getIsp(); + String ispOrganization = response.getOrganization(); + String mobileNetworkCode = response.getMobileNetworkCode(); + String mobileCountryCode = response.getMobileCountryCode(); + Long asn = response.getAutonomousSystemNumber(); + String organizationName = response.getAutonomousSystemOrganization(); + Network network = response.getNetwork(); + + Map data = new HashMap<>(); + for (Database.Property property : this.properties) { + switch (property) { + case IP -> data.put("ip", response.getIpAddress()); + case ASN -> { + if (asn != null) { + data.put("asn", asn); + } + } + case ORGANIZATION_NAME -> { + if (organizationName != null) { + data.put("organization_name", organizationName); + } + } + case NETWORK -> { + if (network != null) { + data.put("network", network.toString()); + } + } + case ISP -> { + if (isp != null) { + data.put("isp", isp); + } + } + case ISP_ORGANIZATION_NAME -> { + if (ispOrganization != null) { + data.put("isp_organization_name", ispOrganization); + } + } + case MOBILE_COUNTRY_CODE -> { + if (mobileCountryCode != null) { + data.put("mobile_country_code", mobileCountryCode); + } + } + case MOBILE_NETWORK_CODE -> { + if (mobileNetworkCode != null) { + data.put("mobile_network_code", mobileNetworkCode); + } + } + } + } + return data; + } + } + + /** + * As an internal detail, the {@code com.maxmind.geoip2.model } classes that are populated by + * {@link Reader#getRecord(InetAddress, Class)} are kinda half-populated and need to go through a second round of construction + * with context from the querying caller. This method gives us a place do that additional binding. Cleverly, the signature + * here matches the constructor for many of these model classes exactly, so an appropriate implementation can 'just' be a method + * reference in some cases (in other cases it needs to be a lambda). + */ + @FunctionalInterface + private interface ResponseBuilder { + RESPONSE build(RESPONSE resp, String address, Network network, List locales); + } + + /** + * The {@link MaxmindIpDataLookups.AbstractBase} is an abstract base implementation of {@link IpDataLookup} that + * provides common functionality for getting a specific kind of {@link AbstractResponse} from a {@link IpDatabase}. + * + * @param the intermediate type of {@link AbstractResponse} + */ + private abstract static class AbstractBase implements IpDataLookup { + + protected final Set properties; + protected final Class clazz; + protected final ResponseBuilder builder; + + AbstractBase(final Set properties, final Class clazz, final ResponseBuilder builder) { + this.properties = Set.copyOf(properties); + this.clazz = clazz; + this.builder = builder; + } + + @Override + public Set getProperties() { + return this.properties; + } + + @Override + public final Map getData(final IpDatabase ipDatabase, final String ipAddress) { + final RESPONSE response = ipDatabase.getResponse(ipAddress, this::lookup); + return (response == null) ? Map.of() : transform(response); + } + + @Nullable + private RESPONSE lookup(final Reader reader, final String ipAddress) throws IOException { + final InetAddress ip = InetAddresses.forString(ipAddress); + final DatabaseRecord record = reader.getRecord(ip, clazz); + final RESPONSE data = record.getData(); + return (data == null) ? null : builder.build(data, NetworkAddress.format(ip), record.getNetwork(), List.of("en")); + } + + /** + * Extract the configured properties from the retrieved response + * @param response the non-null response that was retrieved + * @return a mapping of properties for the ip from the response + */ + protected abstract Map transform(RESPONSE response); + } +} diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java index 83b3d2cfbbc27..7f38a37b43edf 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/ConfigDatabasesTests.java @@ -126,7 +126,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity("89.160.20.128"); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Tumba")); assertThat(cache.count(), equalTo(1)); } @@ -138,7 +138,7 @@ public void testDatabasesUpdateExistingConfDatabase() throws Exception { DatabaseReaderLazyLoader loader = configDatabases.getDatabase("GeoLite2-City.mmdb"); assertThat(loader.getDatabaseType(), equalTo("GeoLite2-City")); - CityResponse cityResponse = loader.getCity("89.160.20.128"); + CityResponse cityResponse = loader.getResponse("89.160.20.128", GeoIpTestUtils::getCity); assertThat(cityResponse.getCity().getName(), equalTo("Linköping")); assertThat(cache.count(), equalTo(1)); }); diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java index f5c3c08579855..793754ec316b2 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.IOUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.RandomDocumentPicks; -import org.elasticsearch.ingest.geoip.Database.Property; import org.elasticsearch.test.ESTestCase; import org.junit.After; import org.junit.Before; @@ -25,7 +24,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; @@ -40,7 +38,9 @@ public class GeoIpProcessorTests extends ESTestCase { - private static final Set ALL_PROPERTIES = Set.of(Property.values()); + private static IpDataLookup ipDataLookupAll(final Database database) { + return IpDataLookupFactories.getMaxmindLookup(database).apply(database.properties()); + } // a temporary directory that mmdb files can be copied to and read from private Path tmpDir; @@ -82,7 +82,7 @@ public void testCity() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -115,7 +115,7 @@ public void testNullValueWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "filename" @@ -137,7 +137,7 @@ public void testNonExistentWithIgnoreMissing() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "filename" @@ -156,7 +156,7 @@ public void testNullWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -178,7 +178,7 @@ public void testNonExistentWithoutIgnoreMissing() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -198,7 +198,7 @@ public void testCity_withIpV6() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -235,7 +235,7 @@ public void testCityWithMissingLocation() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -263,7 +263,7 @@ public void testCountry() throws Exception { loader("GeoLite2-Country.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Country), false, false, "filename" @@ -295,7 +295,7 @@ public void testCountryWithMissingLocation() throws Exception { loader("GeoLite2-Country.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Country), false, false, "filename" @@ -323,7 +323,7 @@ public void testAsn() throws Exception { loader("GeoLite2-ASN.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Asn), false, false, "filename" @@ -354,7 +354,7 @@ public void testAnonymmousIp() throws Exception { loader("GeoIP2-Anonymous-IP-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.AnonymousIp), false, false, "filename" @@ -388,7 +388,7 @@ public void testConnectionType() throws Exception { loader("GeoIP2-Connection-Type-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.ConnectionType), false, false, "filename" @@ -417,7 +417,7 @@ public void testDomain() throws Exception { loader("GeoIP2-Domain-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Domain), false, false, "filename" @@ -446,7 +446,7 @@ public void testEnterprise() throws Exception { loader("GeoIP2-Enterprise-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Enterprise), false, false, "filename" @@ -497,7 +497,7 @@ public void testIsp() throws Exception { loader("GeoIP2-ISP-Test.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.Isp), false, false, "filename" @@ -531,7 +531,7 @@ public void testAddressIsNotInTheDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -555,7 +555,7 @@ public void testInvalid() { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -576,7 +576,7 @@ public void testListAllValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -603,7 +603,7 @@ public void testListPartiallyValid() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -630,7 +630,7 @@ public void testListNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "filename" @@ -650,7 +650,7 @@ public void testListDatabaseReferenceCounting() throws Exception { GeoIpProcessor processor = new GeoIpProcessor(randomAlphaOfLength(10), null, "source_field", () -> { loader.preLookup(); return loader; - }, () -> true, "target_field", ALL_PROPERTIES, false, false, "filename"); + }, () -> true, "target_field", ipDataLookupAll(Database.City), false, false, "filename"); Map document = new HashMap<>(); document.put("source_field", List.of("8.8.8.8", "82.171.64.0")); @@ -678,7 +678,7 @@ public void testListFirstOnly() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -703,7 +703,7 @@ public void testListFirstOnlyNoMatches() throws Exception { loader("GeoLite2-City.mmdb"), () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -725,7 +725,7 @@ public void testInvalidDatabase() throws Exception { loader("GeoLite2-City.mmdb"), () -> false, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, true, "filename" @@ -748,7 +748,7 @@ public void testNoDatabase() throws Exception { () -> null, () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), false, false, "GeoLite2-City" @@ -771,7 +771,7 @@ public void testNoDatabase_ignoreMissing() throws Exception { () -> null, () -> true, "target_field", - ALL_PROPERTIES, + ipDataLookupAll(Database.City), true, false, "GeoLite2-City" diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java index 461983bb24488..160671fd39001 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpTestUtils.java @@ -9,6 +9,13 @@ package org.elasticsearch.ingest.geoip; +import com.maxmind.db.DatabaseRecord; +import com.maxmind.db.Reader; +import com.maxmind.geoip2.model.CityResponse; +import com.maxmind.geoip2.model.CountryResponse; + +import org.elasticsearch.common.CheckedBiFunction; +import org.elasticsearch.common.network.InetAddresses; import org.elasticsearch.core.SuppressForbidden; import java.io.FileNotFoundException; @@ -17,6 +24,7 @@ import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import java.util.Set; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; @@ -58,4 +66,28 @@ public static void copyDefaultDatabases(final Path directory, ConfigDatabases co configDatabases.updateDatabase(directory.resolve(database), true); } } + + /** + * A static city-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

+ * Like this: {@code CityResponse city = loader.getResponse("some.ip.address", GeoIpTestUtils::getCity);} + */ + public static CityResponse getCity(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CityResponse.class); + CityResponse data = record.getData(); + return data == null ? null : new CityResponse(data, ip, record.getNetwork(), List.of("en")); + } + + /** + * A static country-specific responseProvider for use with {@link IpDatabase#getResponse(String, CheckedBiFunction)} in + * tests. + *

+ * Like this: {@code CountryResponse country = loader.getResponse("some.ip.address", GeoIpTestUtils::getCountry);} + */ + public static CountryResponse getCountry(Reader reader, String ip) throws IOException { + DatabaseRecord record = reader.getRecord(InetAddresses.forString(ip), CountryResponse.class); + CountryResponse data = record.getData(); + return data == null ? null : new CountryResponse(data, ip, record.getNetwork(), List.of("en")); + } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java index f1c7d809b98fe..46a34c2cdad56 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MMDBUtilTests.java @@ -116,6 +116,6 @@ public void testDatabaseTypeParsing() throws IOException { } private Database parseDatabaseFromType(String databaseFile) throws IOException { - return Database.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile)), null); + return IpDataLookupFactories.getDatabase(MMDBUtil.getDatabaseType(tmpDir.resolve(databaseFile))); } } diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java index ec05054615bd8..84ea5fd584352 100644 --- a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java +++ b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java @@ -468,36 +468,6 @@ public void testUnknownMaxMindResponseClassess() { ); } - /* - * This tests that this test has a mapping in TYPE_TO_MAX_MIND_CLASS for all MaxMind classes exposed through IpDatabase. - */ - public void testUsedMaxMindResponseClassesAreAccountedFor() { - Set> usedMaxMindResponseClasses = getUsedMaxMindResponseClasses(); - Set> supportedMaxMindClasses = new HashSet<>(TYPE_TO_MAX_MIND_CLASS.values()); - Set> usedButNotSupportedMaxMindResponseClasses = Sets.difference( - usedMaxMindResponseClasses, - supportedMaxMindClasses - ); - assertThat( - "IpDatabase exposes MaxMind response classes that this test does not know what to do with. Add mappings to " - + "TYPE_TO_MAX_MIND_CLASS for the following: " - + usedButNotSupportedMaxMindResponseClasses, - usedButNotSupportedMaxMindResponseClasses, - empty() - ); - Set> supportedButNotUsedMaxMindClasses = Sets.difference( - supportedMaxMindClasses, - usedMaxMindResponseClasses - ); - assertThat( - "This test claims to support MaxMind response classes that are not exposed in IpDatabase. Remove the following from " - + "TYPE_TO_MAX_MIND_CLASS: " - + supportedButNotUsedMaxMindClasses, - supportedButNotUsedMaxMindClasses, - empty() - ); - } - /* * This is the list of field types that causes us to stop recursing. That is, fields of these types are the lowest-level fields that * we care about. @@ -616,23 +586,4 @@ private static String getFormattedList(Set fields) { } return result.toString(); } - - /* - * This returns all AbstractResponse classes that are returned from getter methods on IpDatabase. - */ - private static Set> getUsedMaxMindResponseClasses() { - Set> result = new HashSet<>(); - Method[] methods = IpDatabase.class.getMethods(); - for (Method method : methods) { - if (method.getName().startsWith("get")) { - Class returnType = method.getReturnType(); - try { - result.add(returnType.asSubclass(AbstractResponse.class)); - } catch (ClassCastException ignore) { - // This is not what we were looking for, move on - } - } - } - return result; - } }