Skip to content

Allow empty null values for date and IP field mappers #62487

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Sep 17, 2020
Merged

Allow empty null values for date and IP field mappers #62487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
998736b
Allow empty null values for date and ip field mappers
romseygeek Sep 16, 2020
cdf283c
Only fail null_value parsing for pre-8x indexes
romseygeek Sep 17, 2020
cc6b8e5
formatting
romseygeek Sep 17, 2020
6d995ee
fix settings for base testcase
romseygeek Sep 17, 2020
ed939be
Merge remote-tracking branch 'origin/master' into mapper/date-empty-n…
romseygeek Sep 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 21 additions & 7 deletions server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.geo.ShapeRelation;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.lucene.BytesRefs;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
Expand Down Expand Up @@ -70,6 +72,8 @@
/** A {@link FieldMapper} for dates. */
public final class DateFieldMapper extends ParametrizedFieldMapper {

private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(DateFieldMapper.class);

public static final String CONTENT_TYPE = "date";
public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos";
public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis");
Expand Down Expand Up @@ -200,10 +204,13 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
private final Parameter<Boolean> ignoreMalformed;

private final Resolution resolution;
private final Version indexCreatedVersion;

public Builder(String name, Resolution resolution, DateFormatter dateFormatter, boolean ignoreMalformedByDefault) {
public Builder(String name, Resolution resolution, DateFormatter dateFormatter,
boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
super(name);
this.resolution = resolution;
this.indexCreatedVersion = indexCreatedVersion;
this.ignoreMalformed
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
if (dateFormatter != null) {
Expand Down Expand Up @@ -231,9 +238,14 @@ private Long parseNullValue(DateFieldType fieldType) {
}
try {
return fieldType.parse(nullValue.getValue());
}
catch (Exception e) {
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
} catch (Exception e) {
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
} else {
DEPRECATION_LOGGER.deprecate("date_mapper_null_field", "Error parsing [" + nullValue.getValue()
+ "] as date in [null_value] on field [" + name() + "]); [null_value] will be ignored");
return null;
}
}
}

Expand All @@ -250,12 +262,12 @@ public DateFieldMapper build(BuilderContext context) {

public static final TypeParser MILLIS_PARSER = new TypeParser((n, c) -> {
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), ignoreMalformedByDefault);
return new Builder(n, Resolution.MILLISECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
});

public static final TypeParser NANOS_PARSER = new TypeParser((n, c) -> {
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), ignoreMalformedByDefault);
return new Builder(n, Resolution.NANOSECONDS, c.getDateFormatter(), ignoreMalformedByDefault, c.indexVersionCreated());
});

public static final class DateFieldType extends MappedFieldType {
Expand Down Expand Up @@ -517,6 +529,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
private final Resolution resolution;

private final boolean ignoreMalformedByDefault;
private final Version indexCreatedVersion;

private DateFieldMapper(
String simpleName,
Expand All @@ -537,11 +550,12 @@ private DateFieldMapper(
this.nullValue = nullValue;
this.resolution = resolution;
this.ignoreMalformedByDefault = builder.ignoreMalformed.getDefaultValue();
this.indexCreatedVersion = builder.indexCreatedVersion;
}

@Override
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), resolution, null, ignoreMalformedByDefault).init(this);
return new Builder(simpleName(), resolution, null, ignoreMalformedByDefault, indexCreatedVersion).init(this);
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexableField;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.Version;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.settings.Settings;
Expand Down Expand Up @@ -680,7 +681,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(final ParseContex
if (builder == null) {
boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings());
builder = new DateFieldMapper.Builder(currentFieldName, DateFieldMapper.Resolution.MILLISECONDS,
dateTimeFormatter, ignoreMalformed);
dateTimeFormatter, ignoreMalformed, Version.indexCreated(context.indexSettings().getSettings()));
}
return builder;

Expand Down
54 changes: 37 additions & 17 deletions server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.logging.DeprecationLogger;
import org.elasticsearch.common.network.InetAddresses;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.index.fielddata.IndexFieldData;
import org.elasticsearch.index.fielddata.ScriptDocValues;
import org.elasticsearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
Expand All @@ -55,6 +56,8 @@
/** A {@link FieldMapper} for ip addresses. */
public class IpFieldMapper extends ParametrizedFieldMapper {

private static final DeprecationLogger DEPRECATION_LOGGER = DeprecationLogger.getLogger(IpFieldMapper.class);

public static final String CONTENT_TYPE = "ip";

private static IpFieldMapper toType(FieldMapper in) {
Expand All @@ -68,36 +71,48 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);

private final Parameter<Boolean> ignoreMalformed;
private final Parameter<InetAddress> nullValue = new Parameter<>("null_value", false, () -> null,
(n, c, o) -> o == null ? null : InetAddresses.forString(o.toString()), m -> toType(m).nullValue)
.setSerializer((b, f, v) -> {
if (v == null) {
b.nullField(f);
} else {
b.field(f, InetAddresses.toAddrString(v));
}
}, NetworkAddress::format)
.acceptsNull();
private final Parameter<String> nullValue
= Parameter.stringParam("null_value", false, m -> toType(m).nullValueAsString, null).acceptsNull();

private final Parameter<Map<String, String>> meta = Parameter.metaParam();

private final boolean ignoreMalformedByDefault;
private final Version indexCreatedVersion;

public Builder(String name, boolean ignoreMalformedByDefault) {
public Builder(String name, boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
super(name);
this.ignoreMalformedByDefault = ignoreMalformedByDefault;
this.indexCreatedVersion = indexCreatedVersion;
this.ignoreMalformed
= Parameter.boolParam("ignore_malformed", true, m -> toType(m).ignoreMalformed, ignoreMalformedByDefault);
}

Builder nullValue(InetAddress nullValue) {
Builder nullValue(String nullValue) {
this.nullValue.setValue(nullValue);
return this;
}

private InetAddress parseNullValue() {
String nullValueAsString = nullValue.getValue();
if (nullValueAsString == null) {
return null;
}
try {
return InetAddresses.forString(nullValueAsString);
} catch (Exception e) {
if (indexCreatedVersion.onOrAfter(Version.V_8_0_0)) {
throw new MapperParsingException("Error parsing [null_value] on field [" + name() + "]: " + e.getMessage(), e);
} else {
DEPRECATION_LOGGER.deprecate("ip_mapper_null_field", "Error parsing [" + nullValue.getValue()
+ "] as IP in [null_value] on field [" + name() + "]); [null_value] will be ignored");
return null;
}
}
}

@Override
protected List<Parameter<?>> getParameters() {
return List.of(indexed, hasDocValues, stored, ignoreMalformed, nullValue);
return List.of(indexed, hasDocValues, stored, ignoreMalformed, nullValue, meta);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess the missing "meta" was discovered by the test change?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes!

}

@Override
Expand All @@ -111,7 +126,7 @@ public IpFieldMapper build(BuilderContext context) {

public static final TypeParser PARSER = new TypeParser((n, c) -> {
boolean ignoreMalformedByDefault = IGNORE_MALFORMED_SETTING.get(c.getSettings());
return new Builder(n, ignoreMalformedByDefault);
return new Builder(n, ignoreMalformedByDefault, c.indexVersionCreated());
});

public static final class IpFieldType extends SimpleMappedFieldType {
Expand Down Expand Up @@ -322,9 +337,12 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
private final boolean hasDocValues;
private final boolean stored;
private final boolean ignoreMalformed;

private final InetAddress nullValue;
private final String nullValueAsString;

private final boolean ignoreMalformedByDefault;
private final Version indexCreatedVersion;

private IpFieldMapper(
String simpleName,
Expand All @@ -338,7 +356,9 @@ private IpFieldMapper(
this.hasDocValues = builder.hasDocValues.getValue();
this.stored = builder.stored.getValue();
this.ignoreMalformed = builder.ignoreMalformed.getValue();
this.nullValue = builder.nullValue.getValue();
this.nullValue = builder.parseNullValue();
this.nullValueAsString = builder.nullValue.getValue();
this.indexCreatedVersion = builder.indexCreatedVersion;
}

@Override
Expand Down Expand Up @@ -424,6 +444,6 @@ protected Object parseSourceValue(Object value) {

@Override
public ParametrizedFieldMapper.Builder getMergeBuilder() {
return new Builder(simpleName(), ignoreMalformedByDefault).init(this);
return new Builder(simpleName(), ignoreMalformedByDefault, indexCreatedVersion).init(this);
}
}
3 changes: 2 additions & 1 deletion ...st/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -553,7 +553,8 @@ public void testRolloverClusterStateForDataStream() throws Exception {
try {
Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0));
DateFieldMapper dateFieldMapper
= new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false).build(builderContext);
= new DateFieldMapper.Builder("@timestamp", DateFieldMapper.Resolution.MILLISECONDS, null, false, Version.CURRENT)
.build(builderContext);
MetadataFieldMapper mockedTimestampField = mock(MetadataFieldMapper.class);
when(mockedTimestampField.name()).thenReturn("_data_stream_timestamp");
MappedFieldType mockedTimestampFieldType = mock(MappedFieldType.class);
Expand Down
14 changes: 10 additions & 4 deletions server/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -220,13 +220,18 @@ public void testNanosNullValue() throws IOException {
assertFalse(dvField.fieldType().stored());
}

public void testBadNullValue() {
public void testBadNullValue() throws IOException {

MapperParsingException e = expectThrows(MapperParsingException.class,
() -> createDocumentMapper(fieldMapping(b -> b.field("type", "date").field("null_value", ""))));
() -> createDocumentMapper(Version.V_8_0_0, fieldMapping(b -> b.field("type", "date").field("null_value", "foo"))));

assertThat(e.getMessage(),
equalTo("Failed to parse mapping: Error parsing [null_value] on field [field]: cannot parse empty date"));
equalTo("Failed to parse mapping: Error parsing [null_value] on field [field]: " +
"failed to parse date field [foo] with format [strict_date_optional_time||epoch_millis]"));

createDocumentMapper(Version.V_7_9_0, fieldMapping(b -> b.field("type", "date").field("null_value", "foo")));

assertWarnings("Error parsing [foo] as date in [null_value] on field [field]); [null_value] will be ignored");
}

public void testNullConfigValuesFail() {
Expand Down Expand Up @@ -364,7 +369,8 @@ private DateFieldMapper createMapper(Resolution resolution, String format, Strin
mapping.put("null_value", nullValue);
}

DateFieldMapper.Builder builder = new DateFieldMapper.Builder("field", resolution, null, false);
DateFieldMapper.Builder builder
= new DateFieldMapper.Builder("field", resolution, null, false, Version.CURRENT);
builder.parse("field", null, mapping);
return builder.build(context);
}
Expand Down
Loading