Skip to content

[CCR] Adjust list of dynamic index settings that should be replicated #35195

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 8, 2018
Merged

[CCR] Adjust list of dynamic index settings that should be replicated #35195

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7807e3b
[CCR] Adjust list of dynamic index settings that should be replicated
martijnvg Nov 2, 2018
7dcda6b
Merge remote-tracking branch 'es/master' into ccr_adjust_whitelisted_…
martijnvg Nov 6, 2018
287715c
Merge remote-tracking branch 'es/master' into ccr_adjust_whitelisted_…
martijnvg Nov 7, 2018
2a65635
Merge branch 'master' into pr/35195
jasontedor Nov 8, 2018
6a6f224
Use xor
jasontedor Nov 8, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 58 additions & 4 deletions ...gin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,10 @@
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.block.ClusterBlockLevel;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.routing.UnassignedInfo;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.routing.allocation.decider.EnableAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
import org.elasticsearch.cluster.routing.allocation.decider.ShardsLimitAllocationDecider;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.inject.Inject;
Expand All @@ -28,8 +30,11 @@
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexingSlowLog;
import org.elasticsearch.index.MergePolicyConfig;
import org.elasticsearch.index.MergeSchedulerConfig;
import org.elasticsearch.index.SearchSlowLog;
import org.elasticsearch.index.cache.bitset.BitsetFilterCache;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.shard.ShardId;
import org.elasticsearch.indices.IndicesRequestCache;
Expand Down Expand Up @@ -335,21 +340,37 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
return historyUUIDs.split(",");
}

private static final Set<Setting<?>> WHITE_LISTED_SETTINGS;
/**
* These are settings that are not replicated to the follower index and
* therefor these settings are not validated whether they have the same
* value between leader and follower index.
*
* These dynamic settings don't affect how documents are indexed (affect index time text analysis) and / or
* are inconvenient if they were replicated (e.g. changing number of replicas).
*/
static final Set<Setting<?>> WHITE_LISTED_SETTINGS;
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this should be renamed to NON_REPLICATED_SETTINGS.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agree for a follow-up.


static {
final Set<Setting<?>> whiteListedSettings = new HashSet<>();
whiteListedSettings.add(IndexMetaData.INDEX_NUMBER_OF_REPLICAS_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_AUTO_EXPAND_REPLICAS_SETTING);

whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_EXCLUDE_GROUP_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_INCLUDE_GROUP_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_READ_ONLY_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_WRITE_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_METADATA_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING);
whiteListedSettings.add(IndexMetaData.INDEX_PRIORITY_SETTING);
whiteListedSettings.add(IndexMetaData.SETTING_WAIT_FOR_ACTIVE_SHARDS);

whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING);
whiteListedSettings.add(EnableAllocationDecider.INDEX_ROUTING_ALLOCATION_ENABLE_SETTING);
whiteListedSettings.add(ShardsLimitAllocationDecider.INDEX_TOTAL_SHARDS_PER_NODE_SETTING);
whiteListedSettings.add(MaxRetryAllocationDecider.SETTING_ALLOCATION_MAX_RETRY);
whiteListedSettings.add(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING);

whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING);
whiteListedSettings.add(IndexSettings.MAX_RESULT_WINDOW_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_WARMER_ENABLED_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING);
Expand All @@ -361,6 +382,26 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
whiteListedSettings.add(IndexSettings.QUERY_STRING_ALLOW_LEADING_WILDCARD);
whiteListedSettings.add(IndexSettings.ALLOW_UNMAPPED);
whiteListedSettings.add(IndexSettings.INDEX_SEARCH_IDLE_AFTER);
whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
whiteListedSettings.add(IndexSettings.MAX_SCRIPT_FIELDS_SETTING);
whiteListedSettings.add(IndexSettings.MAX_REGEX_LENGTH_SETTING);
whiteListedSettings.add(IndexSettings.MAX_TERMS_COUNT_SETTING);
whiteListedSettings.add(IndexSettings.MAX_ANALYZED_OFFSET_SETTING);
whiteListedSettings.add(IndexSettings.MAX_DOCVALUE_FIELDS_SEARCH_SETTING);
whiteListedSettings.add(IndexSettings.MAX_TOKEN_COUNT_SETTING);
whiteListedSettings.add(IndexSettings.MAX_SLICES_PER_SCROLL);
whiteListedSettings.add(IndexSettings.MAX_ADJACENCY_MATRIX_FILTERS_SETTING);
whiteListedSettings.add(IndexSettings.DEFAULT_PIPELINE);
whiteListedSettings.add(IndexSettings.INDEX_SEARCH_THROTTLED);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_RETENTION_SIZE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_GENERATION_THRESHOLD_SIZE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING);
whiteListedSettings.add(IndexSettings.INDEX_GC_DELETES_SETTING);
whiteListedSettings.add(IndexSettings.MAX_REFRESH_LISTENERS_PER_SHARD);

whiteListedSettings.add(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING);
whiteListedSettings.add(BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING);

whiteListedSettings.add(SearchSlowLog.INDEX_SEARCH_SLOWLOG_THRESHOLD_FETCH_DEBUG_SETTING);
Expand All @@ -380,7 +421,20 @@ static String[] extractLeaderShardHistoryUUIDs(Map<String, String> ccrIndexMetaD
whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_REFORMAT_SETTING);
whiteListedSettings.add(IndexingSlowLog.INDEX_INDEXING_SLOWLOG_MAX_SOURCE_CHARS_TO_LOG_SETTING);

whiteListedSettings.add(IndexSettings.INDEX_SOFT_DELETES_RETENTION_OPERATIONS_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_COMPOUND_FORMAT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_DELETES_PCT_ALLOWED_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_EXPUNGE_DELETES_ALLOWED_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_FLOOR_SEGMENT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING);
whiteListedSettings.add(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING);

whiteListedSettings.add(MergeSchedulerConfig.AUTO_THROTTLE_SETTING);
whiteListedSettings.add(MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING);
whiteListedSettings.add(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING);
whiteListedSettings.add(EngineConfig.INDEX_CODEC_SETTING);

WHITE_LISTED_SETTINGS = Collections.unmodifiableSet(whiteListedSettings);
}
Expand Down
32 changes: 32 additions & 0 deletions ...cr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@
import org.elasticsearch.Version;
import org.elasticsearch.cluster.metadata.IndexMetaData;
import org.elasticsearch.cluster.metadata.IndexMetaData.State;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.MapperTestUtils;
import org.elasticsearch.index.engine.EngineConfig;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.xpack.ccr.Ccr;
Expand All @@ -21,11 +24,14 @@

import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import static java.util.Collections.singletonMap;
import static org.elasticsearch.xpack.ccr.action.TransportResumeFollowAction.validate;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;

public class TransportResumeFollowActionTests extends ESTestCase {

Expand Down Expand Up @@ -202,6 +208,32 @@ public void testValidation() throws IOException {
validate(request, leaderIMD, followIMD, UUIDs, mapperService);
}
}

public void testDynamicIndexSettingsAreClassified() {
// We should be conscious which dynamic settings are replicated from leader to follower index.
// This is the list of settings that should be replicated:
Set<Setting<?>> replicatedSettings = new HashSet<>();

// These fields need to be replicated otherwise documents that can be indexed in the leader index cannot
// be indexed in the follower index:
replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING);
replicatedSettings.add(MapperService.INDEX_MAPPER_DYNAMIC_SETTING);
replicatedSettings.add(IndexSettings.MAX_NGRAM_DIFF_SETTING);
replicatedSettings.add(IndexSettings.MAX_SHINGLE_DIFF_SETTING);
replicatedSettings.add(EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS);

for (Setting<?> setting : IndexScopedSettings.BUILT_IN_INDEX_SETTINGS) {
if (setting.isDynamic()) {
boolean notReplicated = TransportResumeFollowAction.WHITE_LISTED_SETTINGS.contains(setting);
boolean replicated = replicatedSettings.contains(setting);
assertThat("setting [" + setting.getKey() + "] is not classified as replicated xor not replicated",
notReplicated ^ replicated, is(true));
}
}
}

private static IndexMetaData createIMD(String index,
int numberOfShards,
Expand Down