Skip to content

Commit cc6597d

Browse files
joegallojoegallo
authored
Add support for the 'Domain' database to the geoip processor (#108639)
1 parent 2291008 commit cc6597d

File tree

9 files changed

+132
-16
lines changed

9 files changed

+132
-16
lines changed

‎docs/changelog/108639.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
pr: 108639
2+
summary: Add support for the 'Domain' database to the geoip processor
3+
area: Ingest Node
4+
type: enhancement
5+
issues: []
6+
highlight:
7+
title: Add support for the 'Domain' database to the geoip processor
8+
body: |-
9+
Follow on to #107287 and #107377
10+
11+
Adds support for the ['GeoIP2
12+
Domain'](https://dev.maxmind.com/geoip/docs/databases/domain) database
13+
from MaxMind to the `geoip` processor.
14+
15+
The `geoip` processor will automatically download the [various
16+
'GeoLite2'
17+
databases](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data),
18+
but the 'GeoIP2 Domain' database is not a 'GeoLite2' database -- it's a
19+
commercial database available to those with a suitable license from
20+
MaxMind.
21+
22+
The support that is being added for it in this PR is in line with the
23+
support that we already have for MaxMind's 'GeoIP2 City' and 'GeoIP2
24+
Country' databases -- that is, one would need to arrange their own
25+
download management via some custom endpoint or otherwise arrange for
26+
the relevant file(s) to be in the `$ES_CONFIG/ingest-geoip` directory on
27+
the nodes of the cluster.
28+
notable: true

‎docs/reference/ingest/processors/geoip.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ in `properties`.
5959
* If the GeoIP2 Anonymous IP database is used, then the following fields may be added under the `target_field`: `ip`,
6060
`hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`. The fields actually added
6161
depend on what has been found and which properties were configured in `properties`.
62+
* If the GeoIP2 Domain database is used, then the following fields may be added under the `target_field`: `ip`, and `domain`.
6263
* If the GeoIP2 Enterprise database is used, then the following fields may be added under the `target_field`: `ip`,
6364
`country_iso_code`, `country_name`, `continent_name`, `region_iso_code`, `region_name`, `city_name`, `timezone`, `location`, `asn`,
6465
`organization_name`, `network`, `hosting_provider`, `tor_exit_node`, `anonymous_vpn`, `anonymous`, `public_proxy`, and `residential_proxy`.

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/Database.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ enum Database {
7575
Property.RESIDENTIAL_PROXY
7676
)
7777
),
78+
Domain(Set.of(Property.IP, Property.DOMAIN), Set.of(Property.DOMAIN)),
7879
Enterprise(
7980
Set.of(
8081
Property.IP,
@@ -94,7 +95,8 @@ enum Database {
9495
Property.ANONYMOUS_VPN,
9596
Property.ANONYMOUS,
9697
Property.PUBLIC_PROXY,
97-
Property.RESIDENTIAL_PROXY
98+
Property.RESIDENTIAL_PROXY,
99+
Property.DOMAIN
98100
),
99101
Set.of(
100102
Property.COUNTRY_ISO_CODE,
@@ -111,6 +113,7 @@ enum Database {
111113
private static final String COUNTRY_DB_SUFFIX = "-Country";
112114
private static final String ASN_DB_SUFFIX = "-ASN";
113115
private static final String ANONYMOUS_IP_DB_SUFFIX = "-Anonymous-IP";
116+
private static final String DOMAIN_DB_SUFFIX = "-Domain";
114117
private static final String ENTERPRISE_DB_SUFFIX = "-Enterprise";
115118

116119
/**
@@ -133,6 +136,8 @@ public static Database getDatabase(final String databaseType, final String datab
133136
database = Database.Asn;
134137
} else if (databaseType.endsWith(Database.ANONYMOUS_IP_DB_SUFFIX)) {
135138
database = Database.AnonymousIp;
139+
} else if (databaseType.endsWith(Database.DOMAIN_DB_SUFFIX)) {
140+
database = Database.Domain;
136141
} else if (databaseType.endsWith(Database.ENTERPRISE_DB_SUFFIX)) {
137142
database = Database.Enterprise;
138143
}
@@ -209,7 +214,8 @@ enum Property {
209214
ANONYMOUS_VPN,
210215
ANONYMOUS,
211216
PUBLIC_PROXY,
212-
RESIDENTIAL_PROXY;
217+
RESIDENTIAL_PROXY,
218+
DOMAIN;
213219

214220
/**
215221
* Parses a string representation of a property into an actual Property instance. Not all properties that exist are

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/DatabaseReaderLazyLoader.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import com.maxmind.geoip2.model.AsnResponse;
1717
import com.maxmind.geoip2.model.CityResponse;
1818
import com.maxmind.geoip2.model.CountryResponse;
19+
import com.maxmind.geoip2.model.DomainResponse;
1920
import com.maxmind.geoip2.model.EnterpriseResponse;
2021

2122
import org.apache.logging.log4j.LogManager;
@@ -177,6 +178,12 @@ public AnonymousIpResponse getAnonymousIp(InetAddress ipAddress) {
177178
return getResponse(ipAddress, DatabaseReader::tryAnonymousIp);
178179
}
179180

181+
@Nullable
182+
@Override
183+
public DomainResponse getDomain(InetAddress ipAddress) {
184+
return getResponse(ipAddress, DatabaseReader::tryDomain);
185+
}
186+
180187
@Nullable
181188
@Override
182189
public EnterpriseResponse getEnterprise(InetAddress ipAddress) {

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpDatabase.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import com.maxmind.geoip2.model.AsnResponse;
1313
import com.maxmind.geoip2.model.CityResponse;
1414
import com.maxmind.geoip2.model.CountryResponse;
15+
import com.maxmind.geoip2.model.DomainResponse;
1516
import com.maxmind.geoip2.model.EnterpriseResponse;
1617

1718
import org.elasticsearch.core.Nullable;
@@ -58,6 +59,9 @@ public interface GeoIpDatabase {
5859
@Nullable
5960
AnonymousIpResponse getAnonymousIp(InetAddress ipAddress);
6061

62+
@Nullable
63+
DomainResponse getDomain(InetAddress ipAddress);
64+
6165
@Nullable
6266
EnterpriseResponse getEnterprise(InetAddress ipAddress);
6367

‎modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/GeoIpProcessor.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.maxmind.geoip2.model.AsnResponse;
1414
import com.maxmind.geoip2.model.CityResponse;
1515
import com.maxmind.geoip2.model.CountryResponse;
16+
import com.maxmind.geoip2.model.DomainResponse;
1617
import com.maxmind.geoip2.model.EnterpriseResponse;
1718
import com.maxmind.geoip2.record.City;
1819
import com.maxmind.geoip2.record.Continent;
@@ -175,6 +176,7 @@ private Map<String, Object> getGeoData(GeoIpDatabase geoIpDatabase, String ip) t
175176
case Country -> retrieveCountryGeoData(geoIpDatabase, ipAddress);
176177
case Asn -> retrieveAsnGeoData(geoIpDatabase, ipAddress);
177178
case AnonymousIp -> retrieveAnonymousIpGeoData(geoIpDatabase, ipAddress);
179+
case Domain -> retrieveDomainGeoData(geoIpDatabase, ipAddress);
178180
case Enterprise -> retrieveEnterpriseGeoData(geoIpDatabase, ipAddress);
179181
};
180182
}
@@ -384,6 +386,28 @@ private Map<String, Object> retrieveAnonymousIpGeoData(GeoIpDatabase geoIpDataba
384386
return geoData;
385387
}
386388

389+
private Map<String, Object> retrieveDomainGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
390+
DomainResponse response = geoIpDatabase.getDomain(ipAddress);
391+
if (response == null) {
392+
return Map.of();
393+
}
394+
395+
String domain = response.getDomain();
396+
397+
Map<String, Object> geoData = new HashMap<>();
398+
for (Property property : this.properties) {
399+
switch (property) {
400+
case IP -> geoData.put("ip", NetworkAddress.format(ipAddress));
401+
case DOMAIN -> {
402+
if (domain != null) {
403+
geoData.put("domain", domain);
404+
}
405+
}
406+
}
407+
}
408+
return geoData;
409+
}
410+
387411
private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabase, InetAddress ipAddress) {
388412
EnterpriseResponse response = geoIpDatabase.getEnterprise(ipAddress);
389413
if (response == null) {
@@ -407,6 +431,8 @@ private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas
407431
boolean isPublicProxy = response.getTraits().isPublicProxy();
408432
boolean isResidentialProxy = response.getTraits().isResidentialProxy();
409433

434+
String domain = response.getTraits().getDomain();
435+
410436
Map<String, Object> geoData = new HashMap<>();
411437
for (Property property : this.properties) {
412438
switch (property) {
@@ -500,6 +526,11 @@ private Map<String, Object> retrieveEnterpriseGeoData(GeoIpDatabase geoIpDatabas
500526
case RESIDENTIAL_PROXY -> {
501527
geoData.put("residential_proxy", isResidentialProxy);
502528
}
529+
case DOMAIN -> {
530+
if (domain != null) {
531+
geoData.put("domain", domain);
532+
}
533+
}
503534
}
504535
}
505536
return geoData;

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/GeoIpProcessorTests.java‎

Lines changed: 43 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -336,8 +336,36 @@ public void testAnonymmousIp() throws Exception {
336336
assertThat(geoData.get("residential_proxy"), equalTo(true));
337337
}
338338

339+
public void testDomain() throws Exception {
340+
String ip = "69.219.64.2";
341+
GeoIpProcessor processor = new GeoIpProcessor(
342+
randomAlphaOfLength(10),
343+
null,
344+
"source_field",
345+
loader("/GeoIP2-Domain-Test.mmdb"),
346+
() -> true,
347+
"target_field",
348+
ALL_PROPERTIES,
349+
false,
350+
false,
351+
"filename"
352+
);
353+
354+
Map<String, Object> document = new HashMap<>();
355+
document.put("source_field", ip);
356+
IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
357+
processor.execute(ingestDocument);
358+
359+
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
360+
@SuppressWarnings("unchecked")
361+
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
362+
assertThat(geoData.size(), equalTo(2));
363+
assertThat(geoData.get("ip"), equalTo(ip));
364+
assertThat(geoData.get("domain"), equalTo("ameritech.net"));
365+
}
366+
339367
public void testEnterprise() throws Exception {
340-
String ip = "2.125.160.216";
368+
String ip = "74.209.24.4";
341369
GeoIpProcessor processor = new GeoIpProcessor(
342370
randomAlphaOfLength(10),
343371
null,
@@ -359,26 +387,29 @@ public void testEnterprise() throws Exception {
359387
assertThat(ingestDocument.getSourceAndMetadata().get("source_field"), equalTo(ip));
360388
@SuppressWarnings("unchecked")
361389
Map<String, Object> geoData = (Map<String, Object>) ingestDocument.getSourceAndMetadata().get("target_field");
362-
assertThat(geoData.size(), equalTo(16));
390+
assertThat(geoData.size(), equalTo(19));
363391
assertThat(geoData.get("ip"), equalTo(ip));
364-
assertThat(geoData.get("country_iso_code"), equalTo("GB"));
365-
assertThat(geoData.get("country_name"), equalTo("United Kingdom"));
366-
assertThat(geoData.get("continent_name"), equalTo("Europe"));
367-
assertThat(geoData.get("region_iso_code"), equalTo("GB-WBK"));
368-
assertThat(geoData.get("region_name"), equalTo("West Berkshire"));
369-
assertThat(geoData.get("city_name"), equalTo("Boxford"));
370-
assertThat(geoData.get("timezone"), equalTo("Europe/London"));
392+
assertThat(geoData.get("country_iso_code"), equalTo("US"));
393+
assertThat(geoData.get("country_name"), equalTo("United States"));
394+
assertThat(geoData.get("continent_name"), equalTo("North America"));
395+
assertThat(geoData.get("region_iso_code"), equalTo("US-NY"));
396+
assertThat(geoData.get("region_name"), equalTo("New York"));
397+
assertThat(geoData.get("city_name"), equalTo("Chatham"));
398+
assertThat(geoData.get("timezone"), equalTo("America/New_York"));
371399
Map<String, Object> location = new HashMap<>();
372-
location.put("lat", 51.75);
373-
location.put("lon", -1.25);
400+
location.put("lat", 42.3478);
401+
location.put("lon", -73.5549);
374402
assertThat(geoData.get("location"), equalTo(location));
375-
assertThat(geoData.get("network"), equalTo("2.125.160.216/29"));
403+
assertThat(geoData.get("asn"), equalTo(14671L));
404+
assertThat(geoData.get("organization_name"), equalTo("FairPoint Communications"));
405+
assertThat(geoData.get("network"), equalTo("74.209.16.0/20"));
376406
assertThat(geoData.get("hosting_provider"), equalTo(false));
377407
assertThat(geoData.get("tor_exit_node"), equalTo(false));
378408
assertThat(geoData.get("anonymous_vpn"), equalTo(false));
379409
assertThat(geoData.get("anonymous"), equalTo(false));
380410
assertThat(geoData.get("public_proxy"), equalTo(false));
381411
assertThat(geoData.get("residential_proxy"), equalTo(false));
412+
assertThat(geoData.get("domain"), equalTo("frpt.net"));
382413
}
383414

384415
public void testAddressIsNotInTheDatabase() throws Exception {

‎modules/ingest-geoip/src/test/java/org/elasticsearch/ingest/geoip/MaxMindSupportTests.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,9 @@ public class MaxMindSupportTests extends ESTestCase {
201201
"traits.userType"
202202
);
203203

204+
private static final Set<String> DOMAIN_SUPPORTED_FIELDS = Set.of("domain");
205+
private static final Set<String> DOMAIN_UNSUPPORTED_FIELDS = Set.of("ipAddress", "network");
206+
204207
private static final Set<String> ENTERPRISE_SUPPORTED_FIELDS = Set.of(
205208
"city.name",
206209
"continent.name",
@@ -215,6 +218,7 @@ public class MaxMindSupportTests extends ESTestCase {
215218
"traits.anonymousVpn",
216219
"traits.autonomousSystemNumber",
217220
"traits.autonomousSystemOrganization",
221+
"traits.domain",
218222
"traits.hostingProvider",
219223
"traits.network",
220224
"traits.publicProxy",
@@ -268,7 +272,6 @@ public class MaxMindSupportTests extends ESTestCase {
268272
"traits.anonymousProxy",
269273
"traits.anycast",
270274
"traits.connectionType",
271-
"traits.domain",
272275
"traits.ipAddress",
273276
"traits.isp",
274277
"traits.legitimateProxy",
@@ -290,6 +293,8 @@ public class MaxMindSupportTests extends ESTestCase {
290293
CITY_SUPPORTED_FIELDS,
291294
Database.Country,
292295
COUNTRY_SUPPORTED_FIELDS,
296+
Database.Domain,
297+
DOMAIN_SUPPORTED_FIELDS,
293298
Database.Enterprise,
294299
ENTERPRISE_SUPPORTED_FIELDS
295300
);
@@ -302,6 +307,8 @@ public class MaxMindSupportTests extends ESTestCase {
302307
CITY_UNSUPPORTED_FIELDS,
303308
Database.Country,
304309
COUNTRY_UNSUPPORTED_FIELDS,
310+
Database.Domain,
311+
DOMAIN_UNSUPPORTED_FIELDS,
305312
Database.Enterprise,
306313
ENTERPRISE_UNSUPPORTED_FIELDS
307314
);
@@ -314,13 +321,14 @@ public class MaxMindSupportTests extends ESTestCase {
314321
CityResponse.class,
315322
Database.Country,
316323
CountryResponse.class,
324+
Database.Domain,
325+
DomainResponse.class,
317326
Database.Enterprise,
318327
EnterpriseResponse.class
319328
);
320329

321330
private static final Set<Class<? extends AbstractResponse>> KNOWN_UNSUPPORTED_RESPONSE_CLASSES = Set.of(
322331
ConnectionTypeResponse.class,
323-
DomainResponse.class,
324332
IspResponse.class,
325333
IpRiskResponse.class
326334
);

‎modules/ingest-geoip/src/test/resources/GeoIP2-Domain-Test.mmdb‎

6.3 KB
Binary file not shown.

0 commit comments

Comments
 (0)