Skip to content

Enable soft-deletes by default for 7.0+ indices #38929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Feb 25, 2019
Merged

Enable soft-deletes by default for 7.0+ indices #38929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ba924b7
Enable soft-deletes by default for 7.0+ indices
dnhatn Feb 14, 2019
8f05343
assertions
dnhatn Feb 15, 2019
f3b609d
do not check index_created_version
dnhatn Feb 15, 2019
94ba442
imports
dnhatn Feb 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions server/src/main/java/org/elasticsearch/index/IndexSettings.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -243,9 +243,11 @@ public final class IndexSettings {

/**
* Specifies if the index should use soft-delete instead of hard-delete for update/delete operations.
* Soft-deletes is enabled by default for 7.0+ indices.
*/
public static final Setting<Boolean> INDEX_SOFT_DELETES_SETTING =
Setting.boolSetting("index.soft_deletes.enabled", true, Property.IndexScope, Property.Final);
public static final Setting<Boolean> INDEX_SOFT_DELETES_SETTING = Setting.boolSetting("index.soft_deletes.enabled",
settings -> Boolean.toString(IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(Version.V_7_0_0)),
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if all unit tests are setting this correctly. If the setting is absent, we could perhaps assume that soft-deletes are enabled, instead of disabled.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if we create IndexMetadata without this setting I think it will barf. so I think we are good.

Property.IndexScope, Property.Final);

/**
* Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted
Expand Down
16 changes: 16 additions & 0 deletions server/src/test/java/org/elasticsearch/index/IndexSettingsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,4 +561,20 @@ public void testUpdateSoftDeletesFails() {
Settings.builder(), Settings.builder(), "index"));
assertThat(error.getMessage(), equalTo("final index setting [index.soft_deletes.enabled], not updateable"));
}

public void testSoftDeletesDefaultSetting() {
// enabled by default on 7.0+ or later
{
Version createdVersion = VersionUtils.randomVersionBetween(random(), Version.V_7_0_0, Version.CURRENT);
Settings settings = Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), createdVersion).build();
assertTrue(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings));
}
// disabled by default on the previous versions
{
Version prevVersion = VersionUtils.randomVersionBetween(
random(), Version.V_6_5_0, VersionUtils.getPreviousVersion(Version.V_7_0_0));
Settings settings = Settings.builder().put(IndexMetaData.SETTING_INDEX_VERSION_CREATED.getKey(), prevVersion).build();
assertFalse(IndexSettings.INDEX_SOFT_DELETES_SETTING.get(settings));
}
}
}
3 changes: 2 additions & 1 deletion server/src/test/java/org/elasticsearch/snapshots/SharedClusterSnapshotRestoreIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2210,10 +2210,11 @@ public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedExc
{
SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo")
.setSnapshots("test-2").get().getSnapshots().get(0);
Settings settings = client.admin().indices().prepareGetSettings("test").get().getIndexToSettings().get("test");
List<SnapshotIndexShardStatus> shards = snapshotStatus.getShards();
for (SnapshotIndexShardStatus status : shards) {
// we flush before the snapshot such that we have to process the segments_N files plus the .del file
if (INDEX_SOFT_DELETES_SETTING.get(indexSettings)) {
if (INDEX_SOFT_DELETES_SETTING.get(settings)) {
// soft-delete generates DV files.
assertThat(status.getStats().getProcessedFileCount(), greaterThan(2));
} else {
Expand Down
5 changes: 1 addition & 4 deletions ...ck/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/AutoFollowCoordinator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
Expand Down Expand Up @@ -448,9 +447,7 @@ private void checkAutoFollowPattern(String autoFollowPattenName,
}
} else {
final Settings leaderIndexSettings = remoteMetadata.getIndexSafe(indexToFollow).getSettings();
if (leaderIndexSettings.getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(leaderIndexSettings).onOrAfter(Version.V_7_0_0)) == false) {

if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(leaderIndexSettings) == false) {
String message = String.format(Locale.ROOT, "index [%s] cannot be followed, because soft deletes are not enabled",
indexToFollow.getName());
LOGGER.warn(message);
Expand Down
5 changes: 1 addition & 4 deletions ...plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.message.ParameterizedMessage;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreClusterStateListener;
import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest;
Expand Down Expand Up @@ -124,9 +123,7 @@ private void createFollowerIndex(
listener.onFailure(new IllegalArgumentException("leader index [" + request.getLeaderIndex() + "] does not exist"));
return;
}
// soft deletes are enabled by default on indices created on 7.0.0 or later
if (leaderIndexMetaData.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(leaderIndexMetaData.getSettings()).onOrAfter(Version.V_7_0_0)) == false) {
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(leaderIndexMetaData.getSettings()) == false) {
listener.onFailure(new IllegalArgumentException("leader index [" + request.getLeaderIndex() +
"] does not have soft deletes enabled"));
return;
Expand Down
5 changes: 1 addition & 4 deletions ...gin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

package org.elasticsearch.xpack.ccr.action;

import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
Expand Down Expand Up @@ -214,9 +213,7 @@ static void validate(
"] as history uuid");
}
}
// soft deletes are enabled by default on indices created on 7.0.0 or later
if (leaderIndex.getSettings().getAsBoolean(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(),
IndexMetaData.SETTING_INDEX_VERSION_CREATED.get(leaderIndex.getSettings()).onOrAfter(Version.V_7_0_0)) == false) {
if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(leaderIndex.getSettings()) == false) {
throw new IllegalArgumentException("leader index [" + leaderIndex.getIndex().getName() +
"] does not have soft deletes enabled");
}
Expand Down