Skip to content

Commit

Permalink
Support for translog pruning based on retention leases (#1038)
Browse files Browse the repository at this point in the history 
* Support for translog pruning based on retention leases

Signed-off-by: Sai Kumar <karanas@amazon.com>

* Addressed CR Comments

Signed-off-by: Sai Kumar <karanas@amazon.com>

* Addressed test case issue

Signed-off-by: Sai Kumar <karanas@amazon.com>
  • Loading branch information
@saikaranam-amazon
saikaranam-amazon authored Sep 8, 2021
1 parent 919c5e0 commit a9e1913
Show file tree
Hide file tree
Showing 8 changed files with 220 additions and 11 deletions.
1 change: 1 addition & 0 deletions server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING,
EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING,
IndexSettings.INDEX_FLUSH_AFTER_MERGE_THRESHOLD_SIZE_SETTING,
IndexSettings.INDEX_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING,
IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING,
IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING,
IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING,
Expand Down
46 changes: 43 additions & 3 deletions server/src/main/java/org/opensearch/index/IndexSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,13 @@ public final class IndexSettings {
settings -> Boolean.toString(IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(LegacyESVersion.V_7_0_0)),
Property.IndexScope, Property.Final);

/**
* Specifies if the index translog should prune based on retention leases.
*/
public static final Setting<Boolean> INDEX_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING =
Setting.boolSetting("index.translog.retention_lease.pruning.enabled", false,
Property.IndexScope, Property.Dynamic);

/**
* Controls how many soft-deleted documents will be kept around before being merged away. Keeping more deleted
* documents increases the chance of operation-based recoveries and allows querying a longer history of documents.
Expand All @@ -286,9 +293,11 @@ public final class IndexSettings {
* the chance of ops based recoveries for indices with soft-deletes disabled.
* This setting will be ignored if soft-deletes is used in peer recoveries (default in 7.4).
**/
private static final ByteSizeValue DEFAULT_TRANSLOG_RETENTION_SIZE = new ByteSizeValue(512, ByteSizeUnit.MB);

public static final Setting<ByteSizeValue> INDEX_TRANSLOG_RETENTION_SIZE_SETTING =
Setting.byteSizeSetting("index.translog.retention.size",
settings -> shouldDisableTranslogRetention(settings) ? "-1" : "512MB",
settings -> DEFAULT_TRANSLOG_RETENTION_SIZE.getStringRep(),
Property.Dynamic, Property.IndexScope);

/**
Expand Down Expand Up @@ -389,6 +398,7 @@ public final class IndexSettings {
private final IndexScopedSettings scopedSettings;
private long gcDeletesInMillis = DEFAULT_GC_DELETES.millis();
private final boolean softDeleteEnabled;
private volatile boolean translogPruningByRetentionLease;
private volatile long softDeleteRetentionOperations;

private volatile long retentionLeaseMillis;
Expand Down Expand Up @@ -525,6 +535,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
mergeSchedulerConfig = new MergeSchedulerConfig(this);
gcDeletesInMillis = scopedSettings.get(INDEX_GC_DELETES_SETTING).getMillis();
softDeleteEnabled = version.onOrAfter(LegacyESVersion.V_6_5_0) && scopedSettings.get(INDEX_SOFT_DELETES_SETTING);
translogPruningByRetentionLease = version.onOrAfter(Version.V_1_1_0) &&
softDeleteEnabled &&
scopedSettings.get(INDEX_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING);
softDeleteRetentionOperations = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
retentionLeaseMillis = scopedSettings.get(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING).millis();
warmerEnabled = scopedSettings.get(INDEX_WARMER_ENABLED_SETTING);
Expand Down Expand Up @@ -593,6 +606,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
this::setGenerationThresholdSize);
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_AGE_SETTING, this::setTranslogRetentionAge);
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_SIZE_SETTING, this::setTranslogRetentionSize);
scopedSettings.addSettingsUpdateConsumer(INDEX_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING,
this::setTranslogPruningByRetentionLease);
scopedSettings.addSettingsUpdateConsumer(INDEX_REFRESH_INTERVAL_SETTING, this::setRefreshInterval);
scopedSettings.addSettingsUpdateConsumer(MAX_REFRESH_LISTENERS_PER_SHARD, this::setMaxRefreshListeners);
scopedSettings.addSettingsUpdateConsumer(MAX_ANALYZED_OFFSET_SETTING, this::setHighlightMaxAnalyzedOffset);
Expand Down Expand Up @@ -623,8 +638,14 @@ private void setFlushAfterMergeThresholdSize(ByteSizeValue byteSizeValue) {
this.flushAfterMergeThresholdSize = byteSizeValue;
}

private void setTranslogPruningByRetentionLease(boolean enabled) {
this.translogPruningByRetentionLease = INDEX_SOFT_DELETES_SETTING.get(settings) && enabled;
}

private void setTranslogRetentionSize(ByteSizeValue byteSizeValue) {
if (shouldDisableTranslogRetention(settings) && byteSizeValue.getBytes() >= 0) {
if (shouldDisableTranslogRetention(settings) &&
!shouldPruneTranslogByRetentionLease(settings) &&
byteSizeValue.getBytes() >= 0) {
// ignore the translog retention settings if soft-deletes enabled
this.translogRetentionSize = new ByteSizeValue(-1);
} else {
Expand Down Expand Up @@ -826,7 +847,12 @@ public TimeValue getRefreshInterval() {
* Returns the transaction log retention size which controls how much of the translog is kept around to allow for ops based recoveries
*/
public ByteSizeValue getTranslogRetentionSize() {
assert shouldDisableTranslogRetention(settings) == false || translogRetentionSize.getBytes() == -1L : translogRetentionSize;
if(shouldDisableTranslogRetention(settings) && !shouldPruneTranslogByRetentionLease(settings)) {
return new ByteSizeValue(-1);
}
else if(shouldPruneTranslogByRetentionLease(settings) && translogRetentionSize.getBytes() == -1) {
return DEFAULT_TRANSLOG_RETENTION_SIZE;
}
return translogRetentionSize;
}

Expand Down Expand Up @@ -1071,6 +1097,20 @@ public void setRequiredPipeline(final String requiredPipeline) {
this.requiredPipeline = requiredPipeline;
}

/**
* Returns <code>true</code> if translog ops should be pruned based on retention lease
*/
public boolean shouldPruneTranslogByRetentionLease() {
return translogPruningByRetentionLease;
}

/**
* Returns <code>true</code> if translog ops should be pruned based on retention lease
*/
public static boolean shouldPruneTranslogByRetentionLease(Settings settings) {
return INDEX_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.get(settings);
}

/**
* Returns <code>true</code> if soft-delete is enabled.
*/
Expand Down
5 changes: 5 additions & 0 deletions server/src/main/java/org/opensearch/index/engine/Engine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1849,6 +1849,11 @@ public IndexCommit getIndexCommit() {
}

public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) {
onSettingsChanged(translogRetentionAge, translogRetentionSize, softDeletesRetentionOps, false);
}

public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize,
long softDeletesRetentionOps, boolean translogPruningByRetentionLease) {

}

Expand Down
7 changes: 5 additions & 2 deletions server/src/main/java/org/opensearch/index/engine/InternalEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,8 @@ public InternalEngine(EngineConfig engineConfig) {
final TranslogDeletionPolicy translogDeletionPolicy = new TranslogDeletionPolicy(
engineConfig.getIndexSettings().getTranslogRetentionSize().getBytes(),
engineConfig.getIndexSettings().getTranslogRetentionAge().getMillis(),
engineConfig.getIndexSettings().getTranslogRetentionTotalFiles()
engineConfig.getIndexSettings().getTranslogRetentionTotalFiles(),
engineConfig.retentionLeasesSupplier()
);
store.incRef();
IndexWriter writer = null;
Expand Down Expand Up @@ -2572,7 +2573,8 @@ final void ensureCanFlush() {
}

@Override
public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize, long softDeletesRetentionOps) {
public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue translogRetentionSize,
long softDeletesRetentionOps, boolean translogPruningByRetentionLease) {
mergeScheduler.refreshConfig();
// config().isEnableGcDeletes() or config.getGcDeletesInMillis() may have changed:
maybePruneDeletes();
Expand All @@ -2585,6 +2587,7 @@ public void onSettingsChanged(TimeValue translogRetentionAge, ByteSizeValue tran
final TranslogDeletionPolicy translogDeletionPolicy = translog.getDeletionPolicy();
translogDeletionPolicy.setRetentionAgeInMillis(translogRetentionAge.millis());
translogDeletionPolicy.setRetentionSizeInBytes(translogRetentionSize.getBytes());
translogDeletionPolicy.shouldPruneTranslogByRetentionLease(translogPruningByRetentionLease);
softDeletesPolicy.setRetentionOperations(softDeletesRetentionOps);
}

Expand Down
6 changes: 4 additions & 2 deletions server/src/main/java/org/opensearch/index/shard/IndexShard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1961,8 +1961,10 @@ public void onSettingsChanged() {
final boolean disableTranslogRetention = indexSettings.isSoftDeleteEnabled() && useRetentionLeasesInPeerRecovery;
engineOrNull.onSettingsChanged(
disableTranslogRetention ? TimeValue.MINUS_ONE : indexSettings.getTranslogRetentionAge(),
disableTranslogRetention ? new ByteSizeValue(-1) : indexSettings.getTranslogRetentionSize(),
indexSettings.getSoftDeleteRetentionOperations()
disableTranslogRetention && !indexSettings.shouldPruneTranslogByRetentionLease() ?
new ByteSizeValue(-1) : indexSettings.getTranslogRetentionSize(),
indexSettings.getSoftDeleteRetentionOperations(),
indexSettings.shouldPruneTranslogByRetentionLease()
);
}
}
Expand Down
45 changes: 44 additions & 1 deletion server/src/main/java/org/opensearch/index/translog/TranslogDeletionPolicy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@
import org.apache.lucene.util.Counter;
import org.opensearch.Assertions;
import org.opensearch.common.lease.Releasable;
import org.opensearch.index.seqno.RetentionLease;
import org.opensearch.index.seqno.RetentionLeases;
import org.opensearch.index.seqno.SequenceNumbers;

import java.io.IOException;
Expand All @@ -43,10 +45,12 @@
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Supplier;

public class TranslogDeletionPolicy {

private final Map<Object, RuntimeException> openTranslogRef;
private Supplier<RetentionLeases> retentionLeasesSupplier;

public void assertNoOpenTranslogRefs() {
if (openTranslogRef.isEmpty() == false) {
Expand All @@ -69,6 +73,8 @@ public void assertNoOpenTranslogRefs() {

private int retentionTotalFiles;

private boolean shouldPruneTranslogByRetentionLease;

public TranslogDeletionPolicy(long retentionSizeInBytes, long retentionAgeInMillis, int retentionTotalFiles) {
this.retentionSizeInBytes = retentionSizeInBytes;
this.retentionAgeInMillis = retentionAgeInMillis;
Expand All @@ -80,6 +86,12 @@ public TranslogDeletionPolicy(long retentionSizeInBytes, long retentionAgeInMill
}
}

public TranslogDeletionPolicy(long retentionSizeInBytes, long retentionAgeInMillis, int retentionTotalFiles,
Supplier<RetentionLeases> retentionLeasesSupplier) {
this(retentionSizeInBytes, retentionAgeInMillis, retentionTotalFiles);
this.retentionLeasesSupplier = retentionLeasesSupplier;
}

public synchronized void setLocalCheckpointOfSafeCommit(long newCheckpoint) {
if (newCheckpoint < this.localCheckpointOfSafeCommit) {
throw new IllegalArgumentException("local checkpoint of the safe commit can't go backwards: " +
Expand All @@ -100,6 +112,10 @@ synchronized void setRetentionTotalFiles(int retentionTotalFiles) {
this.retentionTotalFiles = retentionTotalFiles;
}

public synchronized void shouldPruneTranslogByRetentionLease(boolean translogPruneByRetentionLease) {
this.shouldPruneTranslogByRetentionLease = translogPruneByRetentionLease;
}

/**
* acquires the basis generation for a new snapshot. Any translog generation above, and including, the returned generation
* will not be deleted until the returned {@link Releasable} is closed.
Expand Down Expand Up @@ -157,6 +173,12 @@ synchronized long minTranslogGenRequired(List<TranslogReader> readers, TranslogW
long minByLocks = getMinTranslogGenRequiredByLocks();
long minByAge = getMinTranslogGenByAge(readers, writer, retentionAgeInMillis, currentTime());
long minBySize = getMinTranslogGenBySize(readers, writer, retentionSizeInBytes);
long minByRetentionLeasesAndSize = Long.MAX_VALUE;
if(shouldPruneTranslogByRetentionLease) {
// If retention size is specified, size takes precedence.
long minByRetentionLeases = getMinTranslogGenByRetentionLease(readers, writer, retentionLeasesSupplier);
minByRetentionLeasesAndSize = Math.max(minBySize, minByRetentionLeases);
}
final long minByAgeAndSize;
if (minBySize == Long.MIN_VALUE && minByAge == Long.MIN_VALUE) {
// both size and age are disabled;
Expand All @@ -165,7 +187,28 @@ synchronized long minTranslogGenRequired(List<TranslogReader> readers, TranslogW
minByAgeAndSize = Math.max(minByAge, minBySize);
}
long minByNumFiles = getMinTranslogGenByTotalFiles(readers, writer, retentionTotalFiles);
return Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks);
long minByTranslogGenSettings = Math.min(Math.max(minByAgeAndSize, minByNumFiles), minByLocks);
return Math.min(minByTranslogGenSettings, minByRetentionLeasesAndSize);
}

static long getMinTranslogGenByRetentionLease(List<TranslogReader> readers, TranslogWriter writer,
Supplier<RetentionLeases> retentionLeasesSupplier) {
long minGen = writer.getGeneration();
final long minimumRetainingSequenceNumber = retentionLeasesSupplier.get()
.leases()
.stream()
.mapToLong(RetentionLease::retainingSequenceNumber)
.min()
.orElse(Long.MAX_VALUE);

for (int i = readers.size() - 1; i >= 0; i--) {
final TranslogReader reader = readers.get(i);
if(reader.getCheckpoint().minSeqNo <= minimumRetainingSequenceNumber &&
reader.getCheckpoint().maxSeqNo >= minimumRetainingSequenceNumber) {
minGen = Math.min(minGen, reader.getGeneration());
}
}
return minGen;
}

static long getMinTranslogGenBySize(List<TranslogReader> readers, TranslogWriter writer, long retentionSizeInBytes) {
Expand Down
31 changes: 31 additions & 0 deletions server/src/test/java/org/opensearch/index/IndexSettingsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -655,4 +655,35 @@ public void testUpdateTranslogRetentionSettingsWithSoftDeletesDisabled() {
assertThat(indexSettings.getTranslogRetentionAge(), equalTo(ageSetting));
assertThat(indexSettings.getTranslogRetentionSize(), equalTo(sizeSetting));
}

public void testTranslogPruningSettingsWithSoftDeletesEnabled() {
Settings.Builder settings = Settings.builder()
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.V_1_1_0);

ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB);
boolean translogPruningEnabled = randomBoolean();
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), translogPruningEnabled);
IndexMetadata metadata = newIndexMeta("index", settings.build());
IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY);
if(translogPruningEnabled) {
assertTrue(indexSettings.shouldPruneTranslogByRetentionLease());
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes()));
} else {
assertFalse(indexSettings.shouldPruneTranslogByRetentionLease());
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(-1L));
}
}

public void testTranslogPruningSettingsWithSoftDeletesDisabled() {
Settings.Builder settings = Settings.builder()
.put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), false)
.put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT);
boolean translogPruningEnabled = randomBoolean();
ByteSizeValue retentionSize = new ByteSizeValue(512, ByteSizeUnit.MB);
settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_LEASE_PRUNING_ENABLED_SETTING.getKey(), translogPruningEnabled);
IndexMetadata metadata = newIndexMeta("index", settings.build());
IndexSettings indexSettings = new IndexSettings(metadata, Settings.EMPTY);
assertFalse(indexSettings.shouldPruneTranslogByRetentionLease());
assertThat(indexSettings.getTranslogRetentionSize().getBytes(), equalTo(retentionSize.getBytes()));
}
}
Loading

0 comments on commit a9e1913

Please sign in to comment.