Skip to content

Commit 04045a0

Browse files
s1monws1monw
committed
First steps of integrating converted settings into the cluster/index settings infrastructure
The old infa has been removed in this commit such that nothing uses `DynamicSettings` anymore and all index-scoped settings require to be registered before the node has fully started up.
1 parent 211f80f commit 04045a0

File tree

44 files changed

+342
-713
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+342
-713
lines changed

‎core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java

Lines changed: 2 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,9 @@
2222
import org.elasticsearch.cluster.ClusterState;
2323
import org.elasticsearch.cluster.block.ClusterBlocks;
2424
import org.elasticsearch.cluster.metadata.MetaData;
25-
import org.elasticsearch.common.regex.Regex;
2625
import org.elasticsearch.common.settings.ClusterSettings;
2726
import org.elasticsearch.common.settings.Settings;
2827

29-
import java.util.HashSet;
30-
import java.util.Map;
31-
import java.util.Set;
32-
3328
import static org.elasticsearch.cluster.ClusterState.builder;
3429

3530
/**
@@ -57,11 +52,11 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
5752
boolean changed = false;
5853
Settings.Builder transientSettings = Settings.settingsBuilder();
5954
transientSettings.put(currentState.metaData().transientSettings());
60-
changed |= apply(transientToApply, transientSettings, transientUpdates, "transient");
55+
changed |= clusterSettings.applyDynamicSettings(transientToApply, transientSettings, transientUpdates, "transient");
6156

6257
Settings.Builder persistentSettings = Settings.settingsBuilder();
6358
persistentSettings.put(currentState.metaData().persistentSettings());
64-
changed |= apply(persistentToApply, persistentSettings, persistentUpdates, "persistent");
59+
changed |= clusterSettings.applyDynamicSettings(persistentToApply, persistentSettings, persistentUpdates, "persistent");
6560

6661
if (!changed) {
6762
return currentState;
@@ -86,42 +81,5 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
8681
return build;
8782
}
8883

89-
private boolean apply(Settings toApply, Settings.Builder target, Settings.Builder updates, String type) {
90-
boolean changed = false;
91-
final Set<String> toRemove = new HashSet<>();
92-
Settings.Builder settingsBuilder = Settings.settingsBuilder();
93-
for (Map.Entry<String, String> entry : toApply.getAsMap().entrySet()) {
94-
if (entry.getValue() == null) {
95-
toRemove.add(entry.getKey());
96-
} else if (clusterSettings.isLoggerSetting(entry.getKey()) || clusterSettings.hasDynamicSetting(entry.getKey())) {
97-
settingsBuilder.put(entry.getKey(), entry.getValue());
98-
updates.put(entry.getKey(), entry.getValue());
99-
changed = true;
100-
} else {
101-
throw new IllegalArgumentException(type + " setting [" + entry.getKey() + "], not dynamically updateable");
102-
}
103-
104-
}
105-
changed |= applyDeletes(toRemove, target);
106-
target.put(settingsBuilder.build());
107-
return changed;
108-
}
10984

110-
private final boolean applyDeletes(Set<String> deletes, Settings.Builder builder) {
111-
boolean changed = false;
112-
for (String entry : deletes) {
113-
Set<String> keysToRemove = new HashSet<>();
114-
Set<String> keySet = builder.internalMap().keySet();
115-
for (String key : keySet) {
116-
if (Regex.simpleMatch(entry, key)) {
117-
keysToRemove.add(key);
118-
}
119-
}
120-
for (String key : keysToRemove) {
121-
builder.remove(key);
122-
changed = true;
123-
}
124-
}
125-
return changed;
126-
}
12785
}

‎core/src/main/java/org/elasticsearch/cluster/ClusterModule.java

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.elasticsearch.cluster.action.index.NodeIndexDeletedAction;
2424
import org.elasticsearch.cluster.action.index.NodeMappingRefreshAction;
2525
import org.elasticsearch.cluster.action.shard.ShardStateAction;
26-
import org.elasticsearch.cluster.metadata.IndexMetaData;
2726
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2827
import org.elasticsearch.cluster.metadata.IndexTemplateFilter;
2928
import org.elasticsearch.cluster.metadata.MetaDataCreateIndexService;
@@ -55,19 +54,12 @@
5554
import org.elasticsearch.cluster.routing.allocation.decider.SnapshotInProgressAllocationDecider;
5655
import org.elasticsearch.cluster.routing.allocation.decider.ThrottlingAllocationDecider;
5756
import org.elasticsearch.cluster.service.InternalClusterService;
58-
import org.elasticsearch.cluster.settings.DynamicSettings;
59-
import org.elasticsearch.cluster.settings.Validator;
6057
import org.elasticsearch.common.inject.AbstractModule;
6158
import org.elasticsearch.common.logging.ESLogger;
6259
import org.elasticsearch.common.logging.Loggers;
6360
import org.elasticsearch.common.settings.Settings;
6461
import org.elasticsearch.common.util.ExtensionPoint;
6562
import org.elasticsearch.gateway.GatewayAllocator;
66-
import org.elasticsearch.gateway.PrimaryShardAllocator;
67-
import org.elasticsearch.index.IndexSettings;
68-
import org.elasticsearch.index.SearchSlowLog;
69-
import org.elasticsearch.index.settings.IndexDynamicSettings;
70-
import org.elasticsearch.index.MergePolicyConfig;
7163

7264
import java.util.Arrays;
7365
import java.util.Collections;
@@ -98,7 +90,6 @@ public class ClusterModule extends AbstractModule {
9890
SnapshotInProgressAllocationDecider.class));
9991

10092
private final Settings settings;
101-
private final DynamicSettings.Builder indexDynamicSettings = new DynamicSettings.Builder();
10293
private final ExtensionPoint.SelectedType<ShardsAllocator> shardsAllocators = new ExtensionPoint.SelectedType<>("shards_allocator", ShardsAllocator.class);
10394
private final ExtensionPoint.ClassSet<AllocationDecider> allocationDeciders = new ExtensionPoint.ClassSet<>("allocation_decider", AllocationDecider.class, AllocationDeciders.class);
10495
private final ExtensionPoint.ClassSet<IndexTemplateFilter> indexTemplateFilters = new ExtensionPoint.ClassSet<>("index_template_filter", IndexTemplateFilter.class);
@@ -115,10 +106,6 @@ public ClusterModule(Settings settings) {
115106
registerShardsAllocator(ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR, BalancedShardsAllocator.class);
116107
}
117108

118-
public void registerIndexDynamicSetting(String setting, Validator validator) {
119-
indexDynamicSettings.addSetting(setting, validator);
120-
}
121-
122109
public void registerAllocationDecider(Class<? extends AllocationDecider> allocationDecider) {
123110
allocationDeciders.registerExtension(allocationDecider);
124111
}
@@ -133,8 +120,6 @@ public void registerIndexTemplateFilter(Class<? extends IndexTemplateFilter> ind
133120

134121
@Override
135122
protected void configure() {
136-
bind(DynamicSettings.class).annotatedWith(IndexDynamicSettings.class).toInstance(indexDynamicSettings.build());
137-
138123
// bind ShardsAllocator
139124
String shardsAllocatorType = shardsAllocators.bindType(binder(), settings, ClusterModule.SHARDS_ALLOCATOR_TYPE_KEY, ClusterModule.BALANCED_ALLOCATOR);
140125
if (shardsAllocatorType.equals(ClusterModule.EVEN_SHARD_COUNT_ALLOCATOR)) {

‎core/src/main/java/org/elasticsearch/cluster/metadata/IndexMetaData.java

Lines changed: 1 addition & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ public static State fromString(String state) {
181181
public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string";
182182
public static final String SETTING_VERSION_MINIMUM_COMPATIBLE = "index.version.minimum_compatible";
183183
public static final String SETTING_CREATION_DATE = "index.creation_date";
184+
public static final Setting<Long> INDEX_CREATION_DATE_SETTING = Setting.longSetting(SETTING_CREATION_DATE, -1, -1, false, Setting.Scope.INDEX);
184185
public static final String SETTING_PRIORITY = "index.priority";
185186
public static final Setting<Integer> INDEX_PRIORITY_SETTING = Setting.intSetting("index.priority", 1, 0, true, Setting.Scope.INDEX);
186187
public static final String SETTING_CREATION_DATE_STRING = "index.creation_date_string";
@@ -653,10 +654,6 @@ public Builder creationDate(long creationDate) {
653654
return this;
654655
}
655656

656-
public long creationDate() {
657-
return settings.getAsLong(SETTING_CREATION_DATE, -1l);
658-
}
659-
660657
public Builder settings(Settings.Builder settings) {
661658
this.settings = settings.build();
662659
return this;
@@ -671,11 +668,6 @@ public MappingMetaData mapping(String type) {
671668
return mappings.get(type);
672669
}
673670

674-
public Builder removeMapping(String mappingType) {
675-
mappings.remove(mappingType);
676-
return this;
677-
}
678-
679671
public Builder putMapping(String type, String source) throws IOException {
680672
try (XContentParser parser = XContentFactory.xContent(source).createParser(source)) {
681673
putMapping(new MappingMetaData(type, parser.mapOrdered()));
@@ -718,24 +710,11 @@ public Builder putCustom(String type, Custom customIndexMetaData) {
718710
return this;
719711
}
720712

721-
public Builder removeCustom(String type) {
722-
this.customs.remove(type);
723-
return this;
724-
}
725-
726-
public Custom getCustom(String type) {
727-
return this.customs.get(type);
728-
}
729-
730713
public Builder putActiveAllocationIds(int shardId, Set<String> allocationIds) {
731714
activeAllocationIds.put(shardId, new HashSet(allocationIds));
732715
return this;
733716
}
734717

735-
public Set<String> getActiveAllocationIds(int shardId) {
736-
return activeAllocationIds.get(shardId);
737-
}
738-
739718
public long version() {
740719
return this.version;
741720
}

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,9 @@ public void validateIndexName(String index, ClusterState state) {
177177
public void createIndex(final CreateIndexClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
178178
Settings.Builder updatedSettingsBuilder = Settings.settingsBuilder();
179179
updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
180+
for (Map.Entry<String, String> entry : updatedSettingsBuilder.internalMap().entrySet()) {
181+
indexScopeSettings.validate(entry.getKey(), entry.getValue());
182+
}
180183
request.settings(updatedSettingsBuilder.build());
181184

182185
clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]",
@@ -473,7 +476,7 @@ List<String> getIndexSettingsValidationErrors(Settings settings) {
473476
validationErrors.add("custom path [" + customPath + "] is not a sub-path of path.shared_data [" + env.sharedDataFile() + "]");
474477
}
475478
}
476-
//nocommit - this can be removed?
479+
//norelease - this can be removed?
477480
Integer number_of_primaries = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_SHARDS, null);
478481
Integer number_of_replicas = settings.getAsInt(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, null);
479482
if (number_of_primaries != null && number_of_primaries <= 0) {

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

Lines changed: 35 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -34,15 +34,14 @@
3434
import org.elasticsearch.cluster.routing.RoutingTable;
3535
import org.elasticsearch.cluster.routing.allocation.AllocationService;
3636
import org.elasticsearch.cluster.routing.allocation.RoutingAllocation;
37-
import org.elasticsearch.cluster.settings.DynamicSettings;
38-
import org.elasticsearch.common.Booleans;
3937
import org.elasticsearch.common.Priority;
4038
import org.elasticsearch.common.collect.Tuple;
4139
import org.elasticsearch.common.component.AbstractComponent;
4240
import org.elasticsearch.common.inject.Inject;
41+
import org.elasticsearch.common.settings.IndexScopeSettings;
42+
import org.elasticsearch.common.settings.Setting;
4343
import org.elasticsearch.common.settings.Settings;
4444
import org.elasticsearch.common.unit.TimeValue;
45-
import org.elasticsearch.index.settings.IndexDynamicSettings;
4645

4746
import java.util.ArrayList;
4847
import java.util.HashMap;
@@ -63,18 +62,17 @@ public class MetaDataUpdateSettingsService extends AbstractComponent implements
6362

6463
private final AllocationService allocationService;
6564

66-
private final DynamicSettings dynamicSettings;
67-
6865
private final IndexNameExpressionResolver indexNameExpressionResolver;
66+
private final IndexScopeSettings indexScopeSettings;
6967

7068
@Inject
71-
public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService, @IndexDynamicSettings DynamicSettings dynamicSettings, IndexNameExpressionResolver indexNameExpressionResolver) {
69+
public MetaDataUpdateSettingsService(Settings settings, ClusterService clusterService, AllocationService allocationService, IndexScopeSettings indexScopeSettings, IndexNameExpressionResolver indexNameExpressionResolver) {
7270
super(settings);
7371
this.clusterService = clusterService;
7472
this.indexNameExpressionResolver = indexNameExpressionResolver;
7573
this.clusterService.add(this);
7674
this.allocationService = allocationService;
77-
this.dynamicSettings = dynamicSettings;
75+
this.indexScopeSettings = indexScopeSettings;
7876
}
7977

8078
@Override
@@ -147,40 +145,32 @@ public void onFailure(Throwable t) {
147145
public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
148146
Settings.Builder updatedSettingsBuilder = Settings.settingsBuilder();
149147
updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
148+
Settings.Builder settingsForClosedIndices = Settings.builder();
149+
Settings.Builder settingsForOpenIndices = Settings.builder();
150+
Settings.Builder skipppedSettings = Settings.builder();
151+
152+
150153
// never allow to change the number of shards
151-
for (String key : updatedSettingsBuilder.internalMap().keySet()) {
152-
if (key.equals(IndexMetaData.SETTING_NUMBER_OF_SHARDS)) {
154+
for (Map.Entry<String, String> entry : updatedSettingsBuilder.internalMap().entrySet()) {
155+
if (entry.getKey().equals(IndexMetaData.SETTING_NUMBER_OF_SHARDS)) {
153156
listener.onFailure(new IllegalArgumentException("can't change the number of shards for an index"));
154157
return;
155158
}
156-
}
157-
158-
final Settings closeSettings = updatedSettingsBuilder.build();
159-
160-
final Set<String> removedSettings = new HashSet<>();
161-
final Set<String> errors = new HashSet<>();
162-
for (Map.Entry<String, String> setting : updatedSettingsBuilder.internalMap().entrySet()) {
163-
if (!dynamicSettings.hasDynamicSetting(setting.getKey())) {
164-
removedSettings.add(setting.getKey());
165-
} else {
166-
String error = dynamicSettings.validateDynamicSetting(setting.getKey(), setting.getValue(), clusterService.state());
167-
if (error != null) {
168-
errors.add("[" + setting.getKey() + "] - " + error);
169-
}
159+
Setting setting = indexScopeSettings.get(entry.getKey());
160+
if (setting == null) {
161+
throw new IllegalArgumentException("setting [" + entry.getKey() + "] is unknown");
170162
}
171-
}
172-
173-
if (!errors.isEmpty()) {
174-
listener.onFailure(new IllegalArgumentException("can't process the settings: " + errors.toString()));
175-
return;
176-
}
177-
178-
if (!removedSettings.isEmpty()) {
179-
for (String removedSetting : removedSettings) {
180-
updatedSettingsBuilder.remove(removedSetting);
163+
indexScopeSettings.validate(entry.getKey(), entry.getValue());
164+
settingsForClosedIndices.put(entry.getKey(), entry.getValue());
165+
if (setting.isDynamic()) {
166+
settingsForOpenIndices.put(entry.getKey(), entry.getValue());
167+
} else {
168+
skipppedSettings.put(entry.getKey(), entry.getValue());
181169
}
182170
}
183-
final Settings openSettings = updatedSettingsBuilder.build();
171+
final Settings skippedSettigns = skipppedSettings.build();
172+
final Settings closedSettings = settingsForClosedIndices.build();
173+
final Settings openSettings = settingsForOpenIndices.build();
184174

185175
clusterService.submitStateUpdateTask("update-settings",
186176
new AckedClusterStateUpdateTask<ClusterStateUpdateResponse>(Priority.URGENT, request, listener) {
@@ -208,16 +198,16 @@ public ClusterState execute(ClusterState currentState) {
208198
}
209199
}
210200

211-
if (closeIndices.size() > 0 && closeSettings.get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS) != null) {
201+
if (closeIndices.size() > 0 && closedSettings.get(IndexMetaData.SETTING_NUMBER_OF_REPLICAS) != null) {
212202
throw new IllegalArgumentException(String.format(Locale.ROOT,
213203
"Can't update [%s] on closed indices [%s] - can leave index in an unopenable state", IndexMetaData.SETTING_NUMBER_OF_REPLICAS,
214204
closeIndices
215205
));
216206
}
217-
if (!removedSettings.isEmpty() && !openIndices.isEmpty()) {
207+
if (!skippedSettigns.getAsMap().isEmpty() && !openIndices.isEmpty()) {
218208
throw new IllegalArgumentException(String.format(Locale.ROOT,
219209
"Can't update non dynamic settings[%s] for open indices [%s]",
220-
removedSettings,
210+
skippedSettigns.getAsMap().keySet(),
221211
openIndices
222212
));
223213
}
@@ -272,7 +262,7 @@ public ClusterState execute(ClusterState currentState) {
272262

273263
if (!closeIndices.isEmpty()) {
274264
String[] indices = closeIndices.toArray(new String[closeIndices.size()]);
275-
metaDataBuilder.updateSettings(closeSettings, indices);
265+
metaDataBuilder.updateSettings(closedSettings, indices);
276266
}
277267

278268

@@ -281,12 +271,18 @@ public ClusterState execute(ClusterState currentState) {
281271
// now, reroute in case things change that require it (like number of replicas)
282272
RoutingAllocation.Result routingResult = allocationService.reroute(updatedState, "settings update");
283273
updatedState = ClusterState.builder(updatedState).routingResult(routingResult).build();
284-
274+
for (String index : openIndices) {
275+
indexScopeSettings.dryRun(updatedState.metaData().index(index).getSettings());
276+
}
277+
for (String index : closeIndices) {
278+
indexScopeSettings.dryRun(updatedState.metaData().index(index).getSettings());
279+
}
285280
return updatedState;
286281
}
287282
});
288283
}
289284

285+
290286
public void upgradeIndexSettings(final UpgradeSettingsClusterStateUpdateRequest request, final ActionListener<ClusterStateUpdateResponse> listener) {
291287

292288

0 commit comments

Comments
 (0)