Supporting more maxmind fields in the geoip processor (elastic#114268)

carlosdelest · Oct 8, 2024 · 7bbebbd · 7bbebbd
1 parent 3595956
commit 7bbebbd
Show file tree

Hide file tree

Showing 5 changed files with 95 additions and 19 deletions.
diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java
@@ -32,6 +32,7 @@ enum Database {
     City(
         Set.of(
             Property.IP,
+            Property.COUNTRY_IN_EUROPEAN_UNION,
             Property.COUNTRY_ISO_CODE,
             Property.CONTINENT_CODE,
             Property.COUNTRY_NAME,
@@ -41,7 +42,8 @@ enum Database {
             Property.CITY_NAME,
             Property.TIMEZONE,
             Property.LOCATION,
-            Property.POSTAL_CODE
+            Property.POSTAL_CODE,
+            Property.ACCURACY_RADIUS
         ),
         Set.of(
             Property.COUNTRY_ISO_CODE,
@@ -54,7 +56,14 @@ enum Database {
         )
     ),
     Country(
-        Set.of(Property.IP, Property.CONTINENT_CODE, Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE),
+        Set.of(
+            Property.IP,
+            Property.CONTINENT_CODE,
+            Property.CONTINENT_NAME,
+            Property.COUNTRY_NAME,
+            Property.COUNTRY_IN_EUROPEAN_UNION,
+            Property.COUNTRY_ISO_CODE
+        ),
         Set.of(Property.CONTINENT_NAME, Property.COUNTRY_NAME, Property.COUNTRY_ISO_CODE)
     ),
     Asn(
@@ -85,12 +94,15 @@ enum Database {
     Enterprise(
         Set.of(
             Property.IP,
+            Property.COUNTRY_CONFIDENCE,
+            Property.COUNTRY_IN_EUROPEAN_UNION,
             Property.COUNTRY_ISO_CODE,
             Property.COUNTRY_NAME,
             Property.CONTINENT_CODE,
             Property.CONTINENT_NAME,
             Property.REGION_ISO_CODE,
             Property.REGION_NAME,
+            Property.CITY_CONFIDENCE,
             Property.CITY_NAME,
             Property.TIMEZONE,
             Property.LOCATION,
@@ -110,7 +122,9 @@ enum Database {
             Property.MOBILE_NETWORK_CODE,
             Property.USER_TYPE,
             Property.CONNECTION_TYPE,
-            Property.POSTAL_CODE
+            Property.POSTAL_CODE,
+            Property.POSTAL_CONFIDENCE,
+            Property.ACCURACY_RADIUS
         ),
         Set.of(
             Property.COUNTRY_ISO_CODE,
@@ -205,12 +219,15 @@ public Set<Property> parseProperties(@Nullable final List<String> propertyNames)
     enum Property {
 
         IP,
+        COUNTRY_CONFIDENCE,
+        COUNTRY_IN_EUROPEAN_UNION,
         COUNTRY_ISO_CODE,
         COUNTRY_NAME,
         CONTINENT_CODE,
         CONTINENT_NAME,
         REGION_ISO_CODE,
         REGION_NAME,
+        CITY_CONFIDENCE,
         CITY_NAME,
         TIMEZONE,
         LOCATION,
@@ -231,7 +248,9 @@ enum Property {
         CONNECTION_TYPE,
         USER_TYPE,
         TYPE,
-        POSTAL_CODE;
+        POSTAL_CODE,
+        POSTAL_CONFIDENCE,
+        ACCURACY_RADIUS;
 
         /**
          * Parses a string representation of a property into an actual Property instance. Not all properties that exist are

diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/MaxmindIpDataLookups.java
@@ -146,6 +146,12 @@ protected Map<String, Object> transform(final CityResponse response) {
             for (Database.Property property : this.properties) {
                 switch (property) {
                     case IP -> data.put("ip", response.getTraits().getIpAddress());
+                    case COUNTRY_IN_EUROPEAN_UNION -> {
+                        if (country.getIsoCode() != null) {
+                            // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
+                            data.put("country_in_european_union", country.isInEuropeanUnion());
+                        }
+                    }
                     case COUNTRY_ISO_CODE -> {
                         String countryIsoCode = country.getIsoCode();
                         if (countryIsoCode != null) {
@@ -208,6 +214,12 @@ protected Map<String, Object> transform(final CityResponse response) {
                             data.put("location", locationObject);
                         }
                     }
+                    case ACCURACY_RADIUS -> {
+                        Integer accuracyRadius = location.getAccuracyRadius();
+                        if (accuracyRadius != null) {
+                            data.put("accuracy_radius", accuracyRadius);
+                        }
+                    }
                     case POSTAL_CODE -> {
                         if (postal != null && postal.getCode() != null) {
                             data.put("postal_code", postal.getCode());
@@ -261,6 +273,12 @@ protected Map<String, Object> transform(final CountryResponse response) {
             for (Database.Property property : this.properties) {
                 switch (property) {
                     case IP -> data.put("ip", response.getTraits().getIpAddress());
+                    case COUNTRY_IN_EUROPEAN_UNION -> {
+                        if (country.getIsoCode() != null) {
+                            // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
+                            data.put("country_in_european_union", country.isInEuropeanUnion());
+                        }
+                    }
                     case COUNTRY_ISO_CODE -> {
                         String countryIsoCode = country.getIsoCode();
                         if (countryIsoCode != null) {
@@ -359,6 +377,18 @@ protected Map<String, Object> transform(final EnterpriseResponse response) {
             for (Database.Property property : this.properties) {
                 switch (property) {
                     case IP -> data.put("ip", response.getTraits().getIpAddress());
+                    case COUNTRY_CONFIDENCE -> {
+                        Integer countryConfidence = country.getConfidence();
+                        if (countryConfidence != null) {
+                            data.put("country_confidence", countryConfidence);
+                        }
+                    }
+                    case COUNTRY_IN_EUROPEAN_UNION -> {
+                        if (country.getIsoCode() != null) {
+                            // isInEuropeanUnion is a boolean so it can't be null. But it really only makes sense if we have a country
+                            data.put("country_in_european_union", country.isInEuropeanUnion());
+                        }
+                    }
                     case COUNTRY_ISO_CODE -> {
                         String countryIsoCode = country.getIsoCode();
                         if (countryIsoCode != null) {
@@ -399,6 +429,12 @@ protected Map<String, Object> transform(final EnterpriseResponse response) {
                             data.put("region_name", subdivisionName);
                         }
                     }
+                    case CITY_CONFIDENCE -> {
+                        Integer cityConfidence = city.getConfidence();
+                        if (cityConfidence != null) {
+                            data.put("city_confidence", cityConfidence);
+                        }
+                    }
                     case CITY_NAME -> {
                         String cityName = city.getName();
                         if (cityName != null) {
@@ -421,11 +457,23 @@ protected Map<String, Object> transform(final EnterpriseResponse response) {
                             data.put("location", locationObject);
                         }
                     }
+                    case ACCURACY_RADIUS -> {
+                        Integer accuracyRadius = location.getAccuracyRadius();
+                        if (accuracyRadius != null) {
+                            data.put("accuracy_radius", accuracyRadius);
+                        }
+                    }
                     case POSTAL_CODE -> {
                         if (postal != null && postal.getCode() != null) {
                             data.put("postal_code", postal.getCode());
                         }
                     }
+                    case POSTAL_CONFIDENCE -> {
+                        Integer postalConfidence = postal.getConfidence();
+                        if (postalConfidence != null) {
+                            data.put("postal_confidence", postalConfidence);
+                        }
+                    }
                     case ASN -> {
                         if (asn != null) {
                             data.put("asn", asn);

diff --git a/...ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java b/...ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorFactoryTests.java
@@ -195,7 +195,7 @@ public void testBuildWithCountryDbAndAsnFields() {
             equalTo(
                 "[properties] illegal property value ["
                     + asnProperty
-                    + "]. valid values are [IP, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]"
+                    + "]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME]"
             )
         );
     }
@@ -273,9 +273,9 @@ public void testBuildIllegalFieldOption() {
         assertThat(
             e.getMessage(),
             equalTo(
-                "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_ISO_CODE, "
+                "[properties] illegal property value [invalid]. valid values are [IP, COUNTRY_IN_EUROPEAN_UNION, COUNTRY_ISO_CODE, "
                     + "COUNTRY_NAME, CONTINENT_CODE, CONTINENT_NAME, REGION_ISO_CODE, REGION_NAME, CITY_NAME, TIMEZONE, "
-                    + "LOCATION, POSTAL_CODE]"
+                    + "LOCATION, POSTAL_CODE, ACCURACY_RADIUS]"
             )
         );
 

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java
@@ -106,8 +106,9 @@ public void testCity() throws Exception {
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(7));
+        assertThat(geoData.size(), equalTo(9));
         assertThat(geoData.get("ip"), equalTo(ip));
+        assertThat(geoData.get("country_in_european_union"), equalTo(false));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
         assertThat(geoData.get("continent_code"), equalTo("NA"));
@@ -222,8 +223,9 @@ public void testCity_withIpV6() throws Exception {
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(11));
+        assertThat(geoData.size(), equalTo(13));
         assertThat(geoData.get("ip"), equalTo(ip));
+        assertThat(geoData.get("country_in_european_union"), equalTo(false));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
         assertThat(geoData.get("continent_code"), equalTo("NA"));
@@ -233,6 +235,7 @@ public void testCity_withIpV6() throws Exception {
         assertThat(geoData.get("city_name"), equalTo("Homestead"));
         assertThat(geoData.get("timezone"), equalTo("America/New_York"));
         assertThat(geoData.get("location"), equalTo(Map.of("lat", 25.4573d, "lon", -80.4572d)));
+        assertThat(geoData.get("accuracy_radius"), equalTo(50));
         assertThat(geoData.get("postal_code"), equalTo("33035"));
     }
 
@@ -288,8 +291,9 @@ public void testCountry() throws Exception {
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(5));
+        assertThat(geoData.size(), equalTo(6));
         assertThat(geoData.get("ip"), equalTo(ip));
+        assertThat(geoData.get("country_in_european_union"), equalTo(true));
         assertThat(geoData.get("country_iso_code"), equalTo("NL"));
         assertThat(geoData.get("country_name"), equalTo("Netherlands"));
         assertThat(geoData.get("continent_code"), equalTo("EU"));
@@ -471,18 +475,23 @@ public void testEnterprise() throws Exception {
         @SuppressWarnings("unchecked")
         Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
         assertThat(geoData, notNullValue());
-        assertThat(geoData.size(), equalTo(25));
+        assertThat(geoData.size(), equalTo(30));
         assertThat(geoData.get("ip"), equalTo(ip));
+        assertThat(geoData.get("country_confidence"), equalTo(99));
+        assertThat(geoData.get("country_in_european_union"), equalTo(false));
         assertThat(geoData.get("country_iso_code"), equalTo("US"));
         assertThat(geoData.get("country_name"), equalTo("United States"));
         assertThat(geoData.get("continent_code"), equalTo("NA"));
         assertThat(geoData.get("continent_name"), equalTo("North America"));
         assertThat(geoData.get("region_iso_code"), equalTo("US-NY"));
         assertThat(geoData.get("region_name"), equalTo("New York"));
+        assertThat(geoData.get("city_confidence"), equalTo(11));
         assertThat(geoData.get("city_name"), equalTo("Chatham"));
         assertThat(geoData.get("timezone"), equalTo("America/New_York"));
         assertThat(geoData.get("location"), equalTo(Map.of("lat", 42.3478, "lon", -73.5549)));
+        assertThat(geoData.get("accuracy_radius"), equalTo(27));
         assertThat(geoData.get("postal_code"), equalTo("12037"));
+        assertThat(geoData.get("city_confidence"), equalTo(11));
         assertThat(geoData.get("asn"), equalTo(14671L));
         assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications"));
         assertThat(geoData.get("network"), equalTo("74.209.16.0/20"));

diff --git a/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java b/modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java
@@ -78,8 +78,10 @@ public class MaxMindSupportTests extends ESTestCase {
         "city.name",
         "continent.code",
         "continent.name",
+        "country.inEuropeanUnion",
         "country.isoCode",
         "country.name",
+        "location.accuracyRadius",
         "location.latitude",
         "location.longitude",
         "location.timeZone",
@@ -95,14 +97,12 @@ public class MaxMindSupportTests extends ESTestCase {
         "continent.names",
         "country.confidence",
         "country.geoNameId",
-        "country.inEuropeanUnion",
         "country.names",
         "leastSpecificSubdivision.confidence",
         "leastSpecificSubdivision.geoNameId",
         "leastSpecificSubdivision.isoCode",
         "leastSpecificSubdivision.name",
         "leastSpecificSubdivision.names",
-        "location.accuracyRadius",
         "location.averageIncome",
         "location.metroCode",
         "location.populationDensity",
@@ -159,6 +159,7 @@ public class MaxMindSupportTests extends ESTestCase {
 
     private static final Set<String> COUNTRY_SUPPORTED_FIELDS = Set.of(
         "continent.name",
+        "country.inEuropeanUnion",
         "country.isoCode",
         "continent.code",
         "country.name"
@@ -168,7 +169,6 @@ public class MaxMindSupportTests extends ESTestCase {
         "continent.names",
         "country.confidence",
         "country.geoNameId",
-        "country.inEuropeanUnion",
         "country.names",
         "maxMind",
         "registeredCountry.confidence",
@@ -213,17 +213,22 @@ public class MaxMindSupportTests extends ESTestCase {
     private static final Set<String> DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network");
 
     private static final Set<String> ENTERPRISE_SUPPORTED_FIELDS = Set.of(
+        "city.confidence",
         "city.name",
         "continent.code",
         "continent.name",
+        "country.confidence",
+        "country.inEuropeanUnion",
         "country.isoCode",
         "country.name",
+        "location.accuracyRadius",
         "location.latitude",
         "location.longitude",
         "location.timeZone",
         "mostSpecificSubdivision.isoCode",
         "mostSpecificSubdivision.name",
         "postal.code",
+        "postal.confidence",
         "traits.anonymous",
         "traits.anonymousVpn",
         "traits.autonomousSystemNumber",
@@ -242,29 +247,24 @@ public class MaxMindSupportTests extends ESTestCase {
         "traits.userType"
     );
     private static final Set<String> ENTERPRISE_UNSUPPORTED_FIELDS = Set.of(
-        "city.confidence",
         "city.geoNameId",
         "city.names",
         "continent.geoNameId",
         "continent.names",
-        "country.confidence",
         "country.geoNameId",
-        "country.inEuropeanUnion",
         "country.names",
         "leastSpecificSubdivision.confidence",
         "leastSpecificSubdivision.geoNameId",
         "leastSpecificSubdivision.isoCode",
         "leastSpecificSubdivision.name",
         "leastSpecificSubdivision.names",
-        "location.accuracyRadius",
         "location.averageIncome",
         "location.metroCode",
         "location.populationDensity",
         "maxMind",
         "mostSpecificSubdivision.confidence",
         "mostSpecificSubdivision.geoNameId",
         "mostSpecificSubdivision.names",
-        "postal.confidence",
         "registeredCountry.confidence",
         "registeredCountry.geoNameId",
         "registeredCountry.inEuropeanUnion",