Skip to content

Commit 96e0f23

Browse files
martijnvgmartijnvg
committed
Merge branch '6.x' into ccr-6.x
* 6.x: (27 commits) Reduce synchronization on field data cache [Test] #27342 Fix SearchRequests#testValidate Fail queries with scroll that explicitely set request_cache (#27342) add json-processor support for non-map json types (#27335) Upgrade AWS SDK Jackson Databind to 2.6.7.1 Properly format IndexGraveyard deletion date as date (#27362) Stop responding to ping requests before master abdication (#27329) [Test] Fix POI version in packaging tests Added release notes for 6.0.0 Update 6.0.0-beta1.asciidoc Allow affix settings to specify dependencies (#27161) Tests: Improve size regex in documentation test (#26879) reword comment Ensure external refreshes will also refresh internal searcher to minimize segment creation (#27253) Remove unnecessary logger creation for doc values field data [DOCS] Fixed link to docker content Plugins: Add versionless alias to all security policy codebase properties (#26756) [DOCS] Move X-Pack-specific Docker content (#27333) [Test] Fix S3BlobStoreContainerTests.testNumberOfMultiparts() Set minimum_master_nodes to all nodes for REST tests (#27344) ...
2 parents 536c2cc + cd9f26c commit 96e0f23

File tree

96 files changed

+3861
-1815
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

96 files changed

+3861
-1815
lines changed

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,10 @@ class ClusterConfiguration {
6464

6565
/**
6666
* Configuration of the setting <tt>discovery.zen.minimum_master_nodes</tt> on the nodes.
67-
* In case of more than one node, this defaults to (number of nodes / 2) + 1
67+
* In case of more than one node, this defaults to the number of nodes
6868
*/
6969
@Input
70-
Closure<Integer> minimumMasterNodes = { getNumNodes() > 1 ? getNumNodes().intdiv(2) + 1 : -1 }
70+
Closure<Integer> minimumMasterNodes = { getNumNodes() > 1 ? getNumNodes() : -1 }
7171

7272
@Input
7373
String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +

‎core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,23 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
5454
transientSettings.put(currentState.metaData().transientSettings());
5555
changed |= clusterSettings.updateDynamicSettings(transientToApply, transientSettings, transientUpdates, "transient");
5656

57+
5758
Settings.Builder persistentSettings = Settings.builder();
5859
persistentSettings.put(currentState.metaData().persistentSettings());
5960
changed |= clusterSettings.updateDynamicSettings(persistentToApply, persistentSettings, persistentUpdates, "persistent");
6061

6162
final ClusterState clusterState;
6263
if (changed) {
64+
Settings transientFinalSettings = transientSettings.build();
65+
Settings persistentFinalSettings = persistentSettings.build();
66+
// both transient and persistent settings must be consistent by itself we can't allow dependencies to be
67+
// in either of them otherwise a full cluster restart will break the settings validation
68+
clusterSettings.validate(transientFinalSettings, true);
69+
clusterSettings.validate(persistentFinalSettings, true);
70+
6371
MetaData.Builder metaData = MetaData.builder(currentState.metaData())
64-
.persistentSettings(persistentSettings.build())
65-
.transientSettings(transientSettings.build());
72+
.persistentSettings(persistentFinalSettings)
73+
.transientSettings(transientFinalSettings);
6674

6775
ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
6876
boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings())

‎core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ protected void masterOperation(final PutIndexTemplateRequest request, final Clus
7777
}
7878
final Settings.Builder templateSettingsBuilder = Settings.builder();
7979
templateSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
80-
indexScopedSettings.validate(templateSettingsBuilder);
80+
indexScopedSettings.validate(templateSettingsBuilder.build(), true); // templates must be consistent with regards to dependencies
8181
indexTemplateService.putTemplate(new MetaDataIndexTemplateService.PutRequest(cause, request.name())
8282
.patterns(request.patterns())
8383
.order(request.order())

‎core/src/main/java/org/elasticsearch/action/search/SearchRequest.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import org.elasticsearch.common.Strings;
2929
import org.elasticsearch.common.io.stream.StreamInput;
3030
import org.elasticsearch.common.io.stream.StreamOutput;
31+
import org.elasticsearch.common.logging.DeprecationLogger;
32+
import org.elasticsearch.common.logging.Loggers;
3133
import org.elasticsearch.common.unit.TimeValue;
3234
import org.elasticsearch.common.xcontent.ToXContent;
3335
import org.elasticsearch.search.Scroll;
@@ -55,6 +57,7 @@
5557
* @see SearchResponse
5658
*/
5759
public final class SearchRequest extends ActionRequest implements IndicesRequest.Replaceable {
60+
private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(Loggers.getLogger(SearchRequest.class));
5861

5962
private static final ToXContent.Params FORMAT_PARAMS = new ToXContent.MapParams(Collections.singletonMap("pretty", "false"));
6063

@@ -120,6 +123,10 @@ public ActionRequestValidationException validate() {
120123
validationException =
121124
addValidationError("using [from] is not allowed in a scroll context", validationException);
122125
}
126+
if (requestCache != null && requestCache && scroll() != null) {
127+
DEPRECATION_LOGGER.deprecated("Explicitly set [request_cache] for a scroll query is deprecated and will return a 400 " +
128+
"error in future versions");
129+
}
123130
return validationException;
124131
}
125132

@@ -385,7 +392,7 @@ public String getDescription() {
385392
sb.append("], ");
386393
sb.append("search_type[").append(searchType).append("], ");
387394
if (source != null) {
388-
395+
389396
sb.append("source[").append(source.toString(FORMAT_PARAMS)).append("]");
390397
} else {
391398
sb.append("source[]");

‎core/src/main/java/org/elasticsearch/bootstrap/Security.java‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -199,28 +199,28 @@ static Policy readPolicy(URL policyFile, Set<URL> codebases) {
199199
try {
200200
// set codebase properties
201201
for (URL url : codebases) {
202-
String shortName = PathUtils.get(url.toURI()).getFileName().toString();
203-
if (shortName.endsWith(".jar") == false) {
202+
String fileName = PathUtils.get(url.toURI()).getFileName().toString();
203+
if (fileName.endsWith(".jar") == false) {
204204
continue; // tests :(
205205
}
206-
String property = "codebase." + shortName;
207-
if (shortName.startsWith("elasticsearch-rest-client")) {
208-
// The rest client is currently the only example where we have an elasticsearch built artifact
209-
// which needs special permissions in policy files when used. This temporary solution is to
210-
// pass in an extra system property that omits the -version.jar suffix the other properties have.
211-
// That allows the snapshots to reference snapshot builds of the client, and release builds to
212-
// referenced release builds of the client, all with the same grant statements.
213-
final String esVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
214-
final int index = property.indexOf("-" + esVersion + ".jar");
215-
assert index >= 0;
216-
String restClientAlias = property.substring(0, index);
217-
propertiesSet.add(restClientAlias);
218-
System.setProperty(restClientAlias, url.toString());
206+
// We attempt to use a versionless identifier for each codebase. This assumes a specific version
207+
// format in the jar filename. While we cannot ensure all jars in all plugins use this format, nonconformity
208+
// only means policy grants would need to include the entire jar filename as they always have before.
209+
String property = "codebase." + fileName;
210+
String aliasProperty = "codebase." + fileName.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
211+
if (aliasProperty.equals(property) == false) {
212+
propertiesSet.add(aliasProperty);
213+
String previous = System.setProperty(aliasProperty, url.toString());
214+
if (previous != null) {
215+
throw new IllegalStateException("codebase property already set: " + aliasProperty + " -> " + previous +
216+
", cannot set to " + url.toString());
217+
}
219218
}
220219
propertiesSet.add(property);
221220
String previous = System.setProperty(property, url.toString());
222221
if (previous != null) {
223-
throw new IllegalStateException("codebase property already set: " + shortName + "->" + previous);
222+
throw new IllegalStateException("codebase property already set: " + property + " -> " + previous +
223+
", cannot set to " + url.toString());
224224
}
225225
}
226226
return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI()));

‎core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@
3434
import org.elasticsearch.common.xcontent.XContentBuilder;
3535
import org.elasticsearch.common.xcontent.XContentParser;
3636
import org.elasticsearch.index.Index;
37+
import org.joda.time.DateTime;
38+
import org.joda.time.DateTimeZone;
3739

3840
import java.io.IOException;
3941
import java.util.ArrayList;
@@ -432,7 +434,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
432434
builder.startObject();
433435
builder.field(INDEX_KEY);
434436
index.toXContent(builder, params);
435-
builder.timeValueField(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis, TimeUnit.MILLISECONDS);
437+
builder.dateField(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis);
436438
return builder.endObject();
437439
}
438440

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -225,10 +225,9 @@ public void createIndex(final CreateIndexClusterStateUpdateRequest request,
225225
private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request,
226226
final ActionListener<ClusterStateUpdateResponse> listener) {
227227
Settings.Builder updatedSettingsBuilder = Settings.builder();
228-
updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
229-
indexScopedSettings.validate(updatedSettingsBuilder);
230-
request.settings(updatedSettingsBuilder.build());
231-
228+
Settings build = updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build();
229+
indexScopedSettings.validate(build, true); // we do validate here - index setting must be consistent
230+
request.settings(build);
232231
clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]",
233232
new IndexCreationTask(logger, allocationService, request, listener, indicesService, aliasValidator, xContentRegistry, settings,
234233
this::validate));
@@ -425,7 +424,6 @@ public ClusterState execute(ClusterState currentState) throws Exception {
425424
tmpImdBuilder.primaryTerm(shardId, primaryTerm);
426425
}
427426
}
428-
429427
// Set up everything, now locally create the index to see that things are ok, and apply
430428
final IndexMetaData tmpImd = tmpImdBuilder.build();
431429
ActiveShardCount waitForActiveShards = request.waitForActiveShards();

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ private void validate(PutRequest request) {
276276
}
277277

278278
try {
279-
indexScopedSettings.validate(request.settings);
279+
indexScopedSettings.validate(request.settings, true); // templates must be consistent with regards to dependencies
280280
} catch (IllegalArgumentException iae) {
281281
validationErrors.add(iae.getMessage());
282282
for (Throwable t : iae.getSuppressed()) {

‎core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import java.util.Locale;
5555
import java.util.Map;
5656
import java.util.Set;
57+
import java.util.function.Predicate;
5758

5859
import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
5960

@@ -163,7 +164,7 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request
163164
Settings.Builder settingsForOpenIndices = Settings.builder();
164165
final Set<String> skippedSettings = new HashSet<>();
165166

166-
indexScopedSettings.validate(normalizedSettings);
167+
indexScopedSettings.validate(normalizedSettings, false); // don't validate dependencies here we check it below
167168
// never allow to change the number of shards
168169
for (String key : normalizedSettings.keySet()) {
169170
Setting setting = indexScopedSettings.get(key);
@@ -240,7 +241,9 @@ public ClusterState execute(ClusterState currentState) {
240241
if (preserveExisting) {
241242
indexSettings.put(indexMetaData.getSettings());
242243
}
243-
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
244+
Settings finalSettings = indexSettings.build();
245+
indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
246+
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(finalSettings));
244247
}
245248
}
246249
}
@@ -254,7 +257,9 @@ public ClusterState execute(ClusterState currentState) {
254257
if (preserveExisting) {
255258
indexSettings.put(indexMetaData.getSettings());
256259
}
257-
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
260+
Settings finalSettings = indexSettings.build();
261+
indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
262+
metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(finalSettings));
258263
}
259264
}
260265
}

‎core/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java‎

Lines changed: 38 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -264,33 +264,28 @@ public synchronized <T> void addSettingsUpdateConsumer(Setting<T> setting, Consu
264264
}
265265

266266
/**
267-
* Validates that all settings in the builder are registered and valid
267+
* Validates that all given settings are registered and valid
268+
* @param settings the settings to validate
269+
* @param validateDependencies if <code>true</code> settings dependencies are validated as well.
270+
* @see Setting#getSettingsDependencies(String)
268271
*/
269-
public final void validate(Settings.Builder settingsBuilder) {
270-
validate(settingsBuilder.build());
271-
}
272-
273-
/**
274-
* * Validates that all given settings are registered and valid
275-
*/
276-
public final void validate(Settings settings) {
272+
public final void validate(Settings settings, boolean validateDependencies) {
277273
List<RuntimeException> exceptions = new ArrayList<>();
278274
for (String key : settings.keySet()) { // settings iterate in deterministic fashion
279275
try {
280-
validate(key, settings);
276+
validate(key, settings, validateDependencies);
281277
} catch (RuntimeException ex) {
282278
exceptions.add(ex);
283279
}
284280
}
285281
ExceptionsHelper.rethrowAndSuppress(exceptions);
286282
}
287283

288-
289284
/**
290285
* Validates that the setting is valid
291286
*/
292-
public final void validate(String key, Settings settings) {
293-
Setting setting = get(key);
287+
void validate(String key, Settings settings, boolean validateDependencies) {
288+
Setting setting = getRaw(key);
294289
if (setting == null) {
295290
LevensteinDistance ld = new LevensteinDistance();
296291
List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
@@ -315,6 +310,20 @@ public final void validate(String key, Settings settings) {
315310
"settings";
316311
}
317312
throw new IllegalArgumentException(msg);
313+
} else {
314+
Set<String> settingsDependencies = setting.getSettingsDependencies(key);
315+
if (setting.hasComplexMatcher()) {
316+
setting = setting.getConcreteSetting(key);
317+
}
318+
if (validateDependencies && settingsDependencies.isEmpty() == false) {
319+
Set<String> settingKeys = settings.keySet();
320+
for (String requiredSetting : settingsDependencies) {
321+
if (settingKeys.contains(requiredSetting) == false) {
322+
throw new IllegalArgumentException("Missing required setting ["
323+
+ requiredSetting + "] for setting [" + setting.getKey() + "]");
324+
}
325+
}
326+
}
318327
}
319328
setting.get(settings);
320329
}
@@ -375,15 +384,27 @@ default Runnable updater(Settings current, Settings previous) {
375384
/**
376385
* Returns the {@link Setting} for the given key or <code>null</code> if the setting can not be found.
377386
*/
378-
public Setting<?> get(String key) {
387+
public final Setting<?> get(String key) {
388+
Setting<?> raw = getRaw(key);
389+
if (raw == null) {
390+
return null;
391+
} if (raw.hasComplexMatcher()) {
392+
return raw.getConcreteSetting(key);
393+
} else {
394+
return raw;
395+
}
396+
}
397+
398+
private Setting<?> getRaw(String key) {
379399
Setting<?> setting = keySettings.get(key);
380400
if (setting != null) {
381401
return setting;
382402
}
383403
for (Map.Entry<String, Setting<?>> entry : complexMatchers.entrySet()) {
384404
if (entry.getValue().match(key)) {
385405
assert assertMatcher(key, 1);
386-
return entry.getValue().getConcreteSetting(key);
406+
assert entry.getValue().hasComplexMatcher();
407+
return entry.getValue();
387408
}
388409
}
389410
return null;
@@ -513,7 +534,7 @@ private boolean updateSettings(Settings toApply, Settings.Builder target, Settin
513534
} else if (get(key) == null) {
514535
throw new IllegalArgumentException(type + " setting [" + key + "], not recognized");
515536
} else if (isNull == false && canUpdate.test(key)) {
516-
validate(key, toApply);
537+
validate(key, toApply, false); // we might not have a full picture here do to a dependency validation
517538
settingsBuilder.copy(key, toApply);
518539
updates.copy(key, toApply);
519540
changed = true;
@@ -654,7 +675,7 @@ public String setValue(String value) {
654675
* representation. Otherwise <code>false</code>
655676
*/
656677
// TODO this should be replaced by Setting.Property.HIDDEN or something like this.
657-
protected boolean isPrivateSetting(String key) {
678+
public boolean isPrivateSetting(String key) {
658679
return false;
659680
}
660681
}

0 commit comments

Comments
 (0)