diff --git a/build.gradle b/build.gradle index 6517d0292bad8..ec81047e3e6c9 100644 --- a/build.gradle +++ b/build.gradle @@ -131,7 +131,7 @@ task verifyVersions { new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s -> xml = new XmlParser().parse(s) } - Set knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) }) + Set knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }.collect { Version.fromString(it) }) // Limit the known versions to those that should be index compatible, and are not future versions knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy index 09f0ad01578c9..3709805680d7a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy @@ -140,7 +140,7 @@ class PrecommitTasks { configProperties = [ suppressions: checkstyleSuppressions ] - toolVersion = 7.5 + toolVersion = '8.10.1' } project.tasks.withType(Checkstyle) { task -> diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy index 039bce052263c..8dcb862064ec9 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy @@ -149,11 +149,11 @@ public class AntFixture extends AntTask implements Fixture { } // the process is started (has a pid) and is bound to a network interface - // so now wait undil the waitCondition has been met + // so now evaluates if the waitCondition is successful // TODO: change this to a loop? boolean success try { - success = waitCondition(this, ant) == false + success = waitCondition(this, ant) } catch (Exception e) { String msg = "Wait condition caught exception for ${name}" logger.error(msg, e) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java index 1e25a40b0084a..b3075a2fddbd5 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java @@ -19,7 +19,6 @@ package org.elasticsearch.client; -import org.apache.http.Header; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -59,20 +58,6 @@ public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest cl options, ClusterUpdateSettingsResponse::fromXContent, emptySet()); } - /** - * Updates cluster wide specific settings using the Cluster Update Settings API. - *

- * See Cluster Update Settings - * API on elastic.co - * @deprecated Prefer {@link #putSettings(ClusterUpdateSettingsRequest, RequestOptions)} - */ - @Deprecated - public ClusterUpdateSettingsResponse putSettings(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, Header... headers) - throws IOException { - return restHighLevelClient.performRequestAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, - ClusterUpdateSettingsResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API. * See Cluster Update Settings @@ -86,19 +71,6 @@ public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsR restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, options, ClusterUpdateSettingsResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously updates cluster wide specific settings using the Cluster Update Settings API. - *

- * See Cluster Update Settings - * API on elastic.co - * @deprecated Prefer {@link #putSettingsAsync(ClusterUpdateSettingsRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsRequest, - ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings, - ClusterUpdateSettingsResponse::fromXContent, listener, emptySet(), headers); - } /** * Get cluster health using the Cluster Health API. diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java index 5d0376efce5f6..5f85b18091d72 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/IndicesClient.java @@ -19,7 +19,6 @@ package org.elasticsearch.client; -import org.apache.http.Header; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest; import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse; @@ -92,19 +91,6 @@ public DeleteIndexResponse delete(DeleteIndexRequest deleteIndexRequest, Request DeleteIndexResponse::fromXContent, emptySet()); } - /** - * Deletes an index using the Delete Index API. - *

- * See - * Delete Index API on elastic.co - * @deprecated Prefer {@link #delete(DeleteIndexRequest, RequestOptions)} - */ - @Deprecated - public DeleteIndexResponse delete(DeleteIndexRequest deleteIndexRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, - DeleteIndexResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously deletes an index using the Delete Index API. * See @@ -118,19 +104,6 @@ public void deleteAsync(DeleteIndexRequest deleteIndexRequest, RequestOptions op DeleteIndexResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously deletes an index using the Delete Index API. - *

- * See - * Delete Index API on elastic.co - * @deprecated Prefer {@link #deleteAsync(DeleteIndexRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void deleteAsync(DeleteIndexRequest deleteIndexRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(deleteIndexRequest, RequestConverters::deleteIndex, - DeleteIndexResponse::fromXContent, listener, emptySet(), headers); - } - /** * Creates an index using the Create Index API. * See @@ -145,19 +118,6 @@ public CreateIndexResponse create(CreateIndexRequest createIndexRequest, Request CreateIndexResponse::fromXContent, emptySet()); } - /** - * Creates an index using the Create Index API. - *

- * See - * Create Index API on elastic.co - * @deprecated Prefer {@link #create(CreateIndexRequest, RequestOptions)} - */ - @Deprecated - public CreateIndexResponse create(CreateIndexRequest createIndexRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(createIndexRequest, RequestConverters::createIndex, - CreateIndexResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously creates an index using the Create Index API. * See @@ -171,19 +131,6 @@ public void createAsync(CreateIndexRequest createIndexRequest, RequestOptions op CreateIndexResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously creates an index using the Create Index API. - *

- * See - * Create Index API on elastic.co - * @deprecated Prefer {@link #createAsync(CreateIndexRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void createAsync(CreateIndexRequest createIndexRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(createIndexRequest, RequestConverters::createIndex, - CreateIndexResponse::fromXContent, listener, emptySet(), headers); - } - /** * Updates the mappings on an index using the Put Mapping API. * See @@ -198,19 +145,6 @@ public PutMappingResponse putMapping(PutMappingRequest putMappingRequest, Reques PutMappingResponse::fromXContent, emptySet()); } - /** - * Updates the mappings on an index using the Put Mapping API. - *

- * See - * Put Mapping API on elastic.co - * @deprecated Prefer {@link #putMapping(PutMappingRequest, RequestOptions)} - */ - @Deprecated - public PutMappingResponse putMapping(PutMappingRequest putMappingRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(putMappingRequest, RequestConverters::putMapping, - PutMappingResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously updates the mappings on an index using the Put Mapping API. * See @@ -224,20 +158,6 @@ public void putMappingAsync(PutMappingRequest putMappingRequest, RequestOptions PutMappingResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously updates the mappings on an index using the Put Mapping API. - *

- * See - * Put Mapping API on elastic.co - * @deprecated Prefer {@link #putMappingAsync(PutMappingRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void putMappingAsync(PutMappingRequest putMappingRequest, ActionListener listener, - Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(putMappingRequest, RequestConverters::putMapping, - PutMappingResponse::fromXContent, listener, emptySet(), headers); - } - /** * Retrieves the mappings on an index or indices using the Get Mapping API. * See @@ -280,20 +200,6 @@ public IndicesAliasesResponse updateAliases(IndicesAliasesRequest indicesAliases IndicesAliasesResponse::fromXContent, emptySet()); } - /** - * Updates aliases using the Index Aliases API. - *

- * See - * Index Aliases API on elastic.co - * @deprecated {@link #updateAliases(IndicesAliasesRequest, RequestOptions)} - */ - @Deprecated - public IndicesAliasesResponse updateAliases(IndicesAliasesRequest indicesAliasesRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, - IndicesAliasesResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously updates aliases using the Index Aliases API. * See @@ -308,21 +214,6 @@ public void updateAliasesAsync(IndicesAliasesRequest indicesAliasesRequest, Requ IndicesAliasesResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously updates aliases using the Index Aliases API. - *

- * See - * Index Aliases API on elastic.co - * @deprecated Prefer {@link #updateAliasesAsync(IndicesAliasesRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void updateAliasesAsync(IndicesAliasesRequest indicesAliasesRequest, ActionListener listener, - Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(indicesAliasesRequest, RequestConverters::updateAliases, - IndicesAliasesResponse::fromXContent, listener, emptySet(), headers); - } - /** * Opens an index using the Open Index API. * See @@ -337,19 +228,6 @@ public OpenIndexResponse open(OpenIndexRequest openIndexRequest, RequestOptions OpenIndexResponse::fromXContent, emptySet()); } - /** - * Opens an index using the Open Index API. - *

- * See - * Open Index API on elastic.co - * @deprecated Prefer {@link #open(OpenIndexRequest, RequestOptions)} - */ - @Deprecated - public OpenIndexResponse open(OpenIndexRequest openIndexRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(openIndexRequest, RequestConverters::openIndex, - OpenIndexResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously opens an index using the Open Index API. * See @@ -363,19 +241,6 @@ public void openAsync(OpenIndexRequest openIndexRequest, RequestOptions options, OpenIndexResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously opens an index using the Open Index API. - *

- * See - * Open Index API on elastic.co - * @deprecated Prefer {@link #openAsync(OpenIndexRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void openAsync(OpenIndexRequest openIndexRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(openIndexRequest, RequestConverters::openIndex, - OpenIndexResponse::fromXContent, listener, emptySet(), headers); - } - /** * Closes an index using the Close Index API. * See @@ -390,19 +255,6 @@ public CloseIndexResponse close(CloseIndexRequest closeIndexRequest, RequestOpti CloseIndexResponse::fromXContent, emptySet()); } - /** - * Closes an index using the Close Index API. - *

- * See - * Close Index API on elastic.co - * @deprecated Prefer {@link #close(CloseIndexRequest, RequestOptions)} - */ - @Deprecated - public CloseIndexResponse close(CloseIndexRequest closeIndexRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, - CloseIndexResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously closes an index using the Close Index API. * See @@ -417,19 +269,6 @@ public void closeAsync(CloseIndexRequest closeIndexRequest, RequestOptions optio } - /** - * Asynchronously closes an index using the Close Index API. - *

- * See - * Close Index API on elastic.co - * @deprecated Prefer {@link #closeAsync(CloseIndexRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void closeAsync(CloseIndexRequest closeIndexRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(closeIndexRequest, RequestConverters::closeIndex, - CloseIndexResponse::fromXContent, listener, emptySet(), headers); - } - /** * Checks if one or more aliases exist using the Aliases Exist API. * See @@ -444,19 +283,6 @@ public boolean existsAlias(GetAliasesRequest getAliasesRequest, RequestOptions o RestHighLevelClient::convertExistsResponse, emptySet()); } - /** - * Checks if one or more aliases exist using the Aliases Exist API. - *

- * See - * Indices Aliases API on elastic.co - * @deprecated Prefer {@link #existsAlias(GetAliasesRequest, RequestOptions)} - */ - @Deprecated - public boolean existsAlias(GetAliasesRequest getAliasesRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequest(getAliasesRequest, RequestConverters::existsAlias, - RestHighLevelClient::convertExistsResponse, emptySet(), headers); - } - /** * Asynchronously checks if one or more aliases exist using the Aliases Exist API. * See @@ -470,19 +296,6 @@ public void existsAliasAsync(GetAliasesRequest getAliasesRequest, RequestOptions RestHighLevelClient::convertExistsResponse, listener, emptySet()); } - /** - * Asynchronously checks if one or more aliases exist using the Aliases Exist API. - *

- * See - * Indices Aliases API on elastic.co - * @deprecated Prefer {@link #existsAliasAsync(GetAliasesRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void existsAliasAsync(GetAliasesRequest getAliasesRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsync(getAliasesRequest, RequestConverters::existsAlias, - RestHighLevelClient::convertExistsResponse, listener, emptySet(), headers); - } - /** * Refresh one or more indices using the Refresh API. * See Refresh API on elastic.co @@ -496,18 +309,6 @@ public RefreshResponse refresh(RefreshRequest refreshRequest, RequestOptions opt RefreshResponse::fromXContent, emptySet()); } - /** - * Refresh one or more indices using the Refresh API. - *

- * See Refresh API on elastic.co - * @deprecated Prefer {@link #refresh(RefreshRequest, RequestOptions)} - */ - @Deprecated - public RefreshResponse refresh(RefreshRequest refreshRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(refreshRequest, RequestConverters::refresh, RefreshResponse::fromXContent, - emptySet(), headers); - } - /** * Asynchronously refresh one or more indices using the Refresh API. * See Refresh API on elastic.co @@ -520,18 +321,6 @@ public void refreshAsync(RefreshRequest refreshRequest, RequestOptions options, RefreshResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously refresh one or more indices using the Refresh API. - *

- * See Refresh API on elastic.co - * @deprecated Prefer {@link #refreshAsync(RefreshRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void refreshAsync(RefreshRequest refreshRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(refreshRequest, RequestConverters::refresh, RefreshResponse::fromXContent, - listener, emptySet(), headers); - } - /** * Flush one or more indices using the Flush API. * See Flush API on elastic.co @@ -545,18 +334,6 @@ public FlushResponse flush(FlushRequest flushRequest, RequestOptions options) th FlushResponse::fromXContent, emptySet()); } - /** - * Flush one or more indices using the Flush API. - *

- * See Flush API on elastic.co - * @deprecated Prefer {@link #flush(FlushRequest, RequestOptions)} - */ - @Deprecated - public FlushResponse flush(FlushRequest flushRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(flushRequest, RequestConverters::flush, FlushResponse::fromXContent, - emptySet(), headers); - } - /** * Asynchronously flush one or more indices using the Flush API. * See Flush API on elastic.co @@ -569,18 +346,6 @@ public void flushAsync(FlushRequest flushRequest, RequestOptions options, Action FlushResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously flush one or more indices using the Flush API. - *

- * See Flush API on elastic.co - * @deprecated Prefer {@link #flushAsync(FlushRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void flushAsync(FlushRequest flushRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(flushRequest, RequestConverters::flush, FlushResponse::fromXContent, - listener, emptySet(), headers); - } - /** * Initiate a synced flush manually using the synced flush API. * See @@ -651,19 +416,6 @@ public ForceMergeResponse forceMerge(ForceMergeRequest forceMergeRequest, Reques ForceMergeResponse::fromXContent, emptySet()); } - /** - * Force merge one or more indices using the Force Merge API. - *

- * See - * Force Merge API on elastic.co - * @deprecated Prefer {@link #forceMerge(ForceMergeRequest, RequestOptions)} - */ - @Deprecated - public ForceMergeResponse forceMerge(ForceMergeRequest forceMergeRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, - ForceMergeResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously force merge one or more indices using the Force Merge API. * See @@ -677,19 +429,6 @@ public void forceMergeAsync(ForceMergeRequest forceMergeRequest, RequestOptions ForceMergeResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously force merge one or more indices using the Force Merge API. - *

- * See - * Force Merge API on elastic.co - * @deprecated Prefer {@link #forceMergeAsync(ForceMergeRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void forceMergeAsync(ForceMergeRequest forceMergeRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(forceMergeRequest, RequestConverters::forceMerge, - ForceMergeResponse::fromXContent, listener, emptySet(), headers); - } - /** * Clears the cache of one or more indices using the Clear Cache API. * See @@ -705,19 +444,6 @@ public ClearIndicesCacheResponse clearCache(ClearIndicesCacheRequest clearIndice ClearIndicesCacheResponse::fromXContent, emptySet()); } - /** - * Clears the cache of one or more indices using the Clear Cache API. - *

- * See - * Clear Cache API on elastic.co - * @deprecated Prefer {@link #clearCache(ClearIndicesCacheRequest, RequestOptions)} - */ - @Deprecated - public ClearIndicesCacheResponse clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(clearIndicesCacheRequest, RequestConverters::clearCache, - ClearIndicesCacheResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously clears the cache of one or more indices using the Clear Cache API. * See @@ -732,20 +458,6 @@ public void clearCacheAsync(ClearIndicesCacheRequest clearIndicesCacheRequest, R ClearIndicesCacheResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously clears the cache of one or more indices using the Clear Cache API. - *

- * See - * Clear Cache API on elastic.co - * @deprecated Prefer {@link #clearCacheAsync(ClearIndicesCacheRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void clearCacheAsync(ClearIndicesCacheRequest clearIndicesCacheRequest, ActionListener listener, - Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(clearIndicesCacheRequest, RequestConverters::clearCache, - ClearIndicesCacheResponse::fromXContent, listener, emptySet(), headers); - } - /** * Checks if the index (indices) exists or not. * See @@ -765,24 +477,6 @@ public boolean exists(GetIndexRequest request, RequestOptions options) throws IO ); } - /** - * Checks if the index (indices) exists or not. - *

- * See - * Indices Exists API on elastic.co - * @deprecated Prefer {@link #exists(GetIndexRequest, RequestOptions)} - */ - @Deprecated - public boolean exists(GetIndexRequest request, Header... headers) throws IOException { - return restHighLevelClient.performRequest( - request, - RequestConverters::indicesExist, - RestHighLevelClient::convertExistsResponse, - Collections.emptySet(), - headers - ); - } - /** * Asynchronously checks if the index (indices) exists or not. * See @@ -802,25 +496,6 @@ public void existsAsync(GetIndexRequest request, RequestOptions options, ActionL ); } - /** - * Asynchronously checks if the index (indices) exists or not. - *

- * See - * Indices Exists API on elastic.co - * @deprecated Prefer {@link #existsAsync(GetIndexRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void existsAsync(GetIndexRequest request, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsync( - request, - RequestConverters::indicesExist, - RestHighLevelClient::convertExistsResponse, - listener, - Collections.emptySet(), - headers - ); - } - /** * Shrinks an index using the Shrink Index API. * See @@ -835,19 +510,6 @@ public ResizeResponse shrink(ResizeRequest resizeRequest, RequestOptions options ResizeResponse::fromXContent, emptySet()); } - /** - * Shrinks an index using the Shrink Index API. - *

- * See - * Shrink Index API on elastic.co - * @deprecated Prefer {@link #shrink(ResizeRequest, RequestOptions)} - */ - @Deprecated - public ResizeResponse shrink(ResizeRequest resizeRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(resizeRequest, RequestConverters::shrink, ResizeResponse::fromXContent, - emptySet(), headers); - } - /** * Asynchronously shrinks an index using the Shrink index API. * See @@ -861,19 +523,6 @@ public void shrinkAsync(ResizeRequest resizeRequest, RequestOptions options, Act ResizeResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously shrinks an index using the Shrink index API. - *

- * See - * Shrink Index API on elastic.co - * @deprecated Prefer {@link #shrinkAsync(ResizeRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void shrinkAsync(ResizeRequest resizeRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(resizeRequest, RequestConverters::shrink, ResizeResponse::fromXContent, - listener, emptySet(), headers); - } - /** * Splits an index using the Split Index API. * See @@ -888,19 +537,6 @@ public ResizeResponse split(ResizeRequest resizeRequest, RequestOptions options) ResizeResponse::fromXContent, emptySet()); } - /** - * Splits an index using the Split Index API. - *

- * See - * Split Index API on elastic.co - * @deprecated {@link #split(ResizeRequest, RequestOptions)} - */ - @Deprecated - public ResizeResponse split(ResizeRequest resizeRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(resizeRequest, RequestConverters::split, ResizeResponse::fromXContent, - emptySet(), headers); - } - /** * Asynchronously splits an index using the Split Index API. * See @@ -914,19 +550,6 @@ public void splitAsync(ResizeRequest resizeRequest, RequestOptions options, Acti ResizeResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously splits an index using the Split Index API. - *

- * See - * Split Index API on elastic.co - * @deprecated Prefer {@link #splitAsync(ResizeRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void splitAsync(ResizeRequest resizeRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(resizeRequest, RequestConverters::split, ResizeResponse::fromXContent, - listener, emptySet(), headers); - } - /** * Rolls over an index using the Rollover Index API. * See @@ -941,19 +564,6 @@ public RolloverResponse rollover(RolloverRequest rolloverRequest, RequestOptions RolloverResponse::fromXContent, emptySet()); } - /** - * Rolls over an index using the Rollover Index API. - *

- * See - * Rollover Index API on elastic.co - * @deprecated Prefer {@link #rollover(RolloverRequest, RequestOptions)} - */ - @Deprecated - public RolloverResponse rollover(RolloverRequest rolloverRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(rolloverRequest, RequestConverters::rollover, - RolloverResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously rolls over an index using the Rollover Index API. * See @@ -967,19 +577,6 @@ public void rolloverAsync(RolloverRequest rolloverRequest, RequestOptions option RolloverResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously rolls over an index using the Rollover Index API. - *

- * See - * Rollover Index API on elastic.co - * @deprecated Prefer {@link #rolloverAsync(RolloverRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void rolloverAsync(RolloverRequest rolloverRequest, ActionListener listener, Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(rolloverRequest, RequestConverters::rollover, RolloverResponse::fromXContent, - listener, emptySet(), headers); - } - /** * Gets one or more aliases using the Get Index Aliases API. * See Indices Aliases API on @@ -1021,19 +618,6 @@ public UpdateSettingsResponse putSettings(UpdateSettingsRequest updateSettingsRe UpdateSettingsResponse::fromXContent, emptySet()); } - /** - * Updates specific index level settings using the Update Indices Settings API. - *

- * See Update Indices Settings - * API on elastic.co - * @deprecated Prefer {@link #putSettings(UpdateSettingsRequest, RequestOptions)} - */ - @Deprecated - public UpdateSettingsResponse putSettings(UpdateSettingsRequest updateSettingsRequest, Header... headers) throws IOException { - return restHighLevelClient.performRequestAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, - UpdateSettingsResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously updates specific index level settings using the Update Indices Settings API. * See Update Indices Settings @@ -1048,20 +632,6 @@ public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, Reques UpdateSettingsResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously updates specific index level settings using the Update Indices Settings API. - *

- * See Update Indices Settings - * API on elastic.co - * @deprecated Prefer {@link #putSettingsAsync(UpdateSettingsRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public void putSettingsAsync(UpdateSettingsRequest updateSettingsRequest, ActionListener listener, - Header... headers) { - restHighLevelClient.performRequestAsyncAndParseEntity(updateSettingsRequest, RequestConverters::indexPutSettings, - UpdateSettingsResponse::fromXContent, listener, emptySet(), headers); - } - /** * Puts an index template using the Index Templates API. * See Index Templates API diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index e2b61dc41deb0..93bf6a1a19881 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -721,7 +721,7 @@ static Request clusterHealth(ClusterHealthRequest healthRequest) { .withWaitForStatus(healthRequest.waitForStatus()) .withWaitForNoRelocatingShards(healthRequest.waitForNoRelocatingShards()) .withWaitForNoInitializingShards(healthRequest.waitForNoInitializingShards()) - .withWaitForActiveShards(healthRequest.waitForActiveShards()) + .withWaitForActiveShards(healthRequest.waitForActiveShards(), ActiveShardCount.NONE) .withWaitForNodes(healthRequest.waitForNodes()) .withWaitForEvents(healthRequest.waitForEvents()) .withTimeout(healthRequest.timeout()) @@ -1047,7 +1047,11 @@ Params withVersionType(VersionType versionType) { } Params withWaitForActiveShards(ActiveShardCount activeShardCount) { - if (activeShardCount != null && activeShardCount != ActiveShardCount.DEFAULT) { + return withWaitForActiveShards(activeShardCount, ActiveShardCount.DEFAULT); + } + + Params withWaitForActiveShards(ActiveShardCount activeShardCount, ActiveShardCount defaultActiveShardCount) { + if (activeShardCount != null && activeShardCount != defaultActiveShardCount) { return putParam("wait_for_active_shards", activeShardCount.toString().toLowerCase(Locale.ROOT)); } return this; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 0084ce0f90d74..536b85925a4ba 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -296,17 +296,6 @@ public final BulkResponse bulk(BulkRequest bulkRequest, RequestOptions options) return performRequestAndParseEntity(bulkRequest, RequestConverters::bulk, options, BulkResponse::fromXContent, emptySet()); } - /** - * Executes a bulk request using the Bulk API. - * - * See Bulk API on elastic.co - * @deprecated Prefer {@link #bulk(BulkRequest, RequestOptions)} - */ - @Deprecated - public final BulkResponse bulk(BulkRequest bulkRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(bulkRequest, RequestConverters::bulk, BulkResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously executes a bulk request using the Bulk API. * See Bulk API on elastic.co @@ -318,17 +307,6 @@ public final void bulkAsync(BulkRequest bulkRequest, RequestOptions options, Act performRequestAsyncAndParseEntity(bulkRequest, RequestConverters::bulk, options, BulkResponse::fromXContent, listener, emptySet()); } - /** - * Asynchronously executes a bulk request using the Bulk API. - * - * See Bulk API on elastic.co - * @deprecated Prefer {@link #bulkAsync(BulkRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void bulkAsync(BulkRequest bulkRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(bulkRequest, RequestConverters::bulk, BulkResponse::fromXContent, listener, emptySet(), headers); - } - /** * Pings the remote Elasticsearch cluster and returns true if the ping succeeded, false otherwise * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized @@ -340,16 +318,6 @@ public final boolean ping(RequestOptions options) throws IOException { emptySet()); } - /** - * Pings the remote Elasticsearch cluster and returns true if the ping succeeded, false otherwise - * @deprecated Prefer {@link #ping(RequestOptions)} - */ - @Deprecated - public final boolean ping(Header... headers) throws IOException { - return performRequest(new MainRequest(), (request) -> RequestConverters.ping(), RestHighLevelClient::convertExistsResponse, - emptySet(), headers); - } - /** * Get the cluster info otherwise provided when sending an HTTP request to '/' * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized @@ -361,16 +329,6 @@ public final MainResponse info(RequestOptions options) throws IOException { MainResponse::fromXContent, emptySet()); } - /** - * Get the cluster info otherwise provided when sending an HTTP request to port 9200 - * @deprecated Prefer {@link #info(RequestOptions)} - */ - @Deprecated - public final MainResponse info(Header... headers) throws IOException { - return performRequestAndParseEntity(new MainRequest(), (request) -> RequestConverters.info(), - MainResponse::fromXContent, emptySet(), headers); - } - /** * Retrieves a document by id using the Get API. * See Get API on elastic.co @@ -383,17 +341,6 @@ public final GetResponse get(GetRequest getRequest, RequestOptions options) thro return performRequestAndParseEntity(getRequest, RequestConverters::get, options, GetResponse::fromXContent, singleton(404)); } - /** - * Retrieves a document by id using the Get API. - * - * See Get API on elastic.co - * @deprecated Prefer {@link #get(GetRequest, RequestOptions)} - */ - @Deprecated - public final GetResponse get(GetRequest getRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(getRequest, RequestConverters::get, GetResponse::fromXContent, singleton(404), headers); - } - /** * Asynchronously retrieves a document by id using the Get API. * See Get API on elastic.co @@ -406,18 +353,6 @@ public final void getAsync(GetRequest getRequest, RequestOptions options, Action singleton(404)); } - /** - * Asynchronously retrieves a document by id using the Get API. - * - * See Get API on elastic.co - * @deprecated Prefer {@link #getAsync(GetRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void getAsync(GetRequest getRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(getRequest, RequestConverters::get, GetResponse::fromXContent, listener, - singleton(404), headers); - } - /** * Retrieves multiple documents by id using the Multi Get API. * See Multi Get API on elastic.co @@ -431,18 +366,6 @@ public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, RequestO singleton(404)); } - /** - * Retrieves multiple documents by id using the Multi Get API. - * - * See Multi Get API on elastic.co - * @deprecated Prefer {@link #multiGet(MultiGetRequest, RequestOptions)} - */ - @Deprecated - public final MultiGetResponse multiGet(MultiGetRequest multiGetRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(multiGetRequest, RequestConverters::multiGet, MultiGetResponse::fromXContent, - singleton(404), headers); - } - /** * Asynchronously retrieves multiple documents by id using the Multi Get API. * See Multi Get API on elastic.co @@ -455,18 +378,6 @@ public final void multiGetAsync(MultiGetRequest multiGetRequest, RequestOptions singleton(404)); } - /** - * Asynchronously retrieves multiple documents by id using the Multi Get API. - * - * See Multi Get API on elastic.co - * @deprecated Prefer {@link #multiGetAsync(MultiGetRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void multiGetAsync(MultiGetRequest multiGetRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(multiGetRequest, RequestConverters::multiGet, MultiGetResponse::fromXContent, listener, - singleton(404), headers); - } - /** * Checks for the existence of a document. Returns true if it exists, false otherwise. * See Get API on elastic.co @@ -479,17 +390,6 @@ public final boolean exists(GetRequest getRequest, RequestOptions options) throw return performRequest(getRequest, RequestConverters::exists, options, RestHighLevelClient::convertExistsResponse, emptySet()); } - /** - * Checks for the existence of a document. Returns true if it exists, false otherwise. - * - * See Get API on elastic.co - * @deprecated Prefer {@link #exists(GetRequest, RequestOptions)} - */ - @Deprecated - public final boolean exists(GetRequest getRequest, Header... headers) throws IOException { - return performRequest(getRequest, RequestConverters::exists, RestHighLevelClient::convertExistsResponse, emptySet(), headers); - } - /** * Asynchronously checks for the existence of a document. Returns true if it exists, false otherwise. * See Get API on elastic.co @@ -502,18 +402,6 @@ public final void existsAsync(GetRequest getRequest, RequestOptions options, Act emptySet()); } - /** - * Asynchronously checks for the existence of a document. Returns true if it exists, false otherwise. - * - * See Get API on elastic.co - * @deprecated Prefer {@link #existsAsync(GetRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void existsAsync(GetRequest getRequest, ActionListener listener, Header... headers) { - performRequestAsync(getRequest, RequestConverters::exists, RestHighLevelClient::convertExistsResponse, listener, - emptySet(), headers); - } - /** * Index a document using the Index API. * See Index API on elastic.co @@ -526,17 +414,6 @@ public final IndexResponse index(IndexRequest indexRequest, RequestOptions optio return performRequestAndParseEntity(indexRequest, RequestConverters::index, options, IndexResponse::fromXContent, emptySet()); } - /** - * Index a document using the Index API. - * - * See Index API on elastic.co - * @deprecated Prefer {@link #index(IndexRequest, RequestOptions)} - */ - @Deprecated - public final IndexResponse index(IndexRequest indexRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(indexRequest, RequestConverters::index, IndexResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously index a document using the Index API. * See Index API on elastic.co @@ -549,18 +426,6 @@ public final void indexAsync(IndexRequest indexRequest, RequestOptions options, emptySet()); } - /** - * Asynchronously index a document using the Index API. - * - * See Index API on elastic.co - * @deprecated Prefer {@link #indexAsync(IndexRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void indexAsync(IndexRequest indexRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(indexRequest, RequestConverters::index, IndexResponse::fromXContent, listener, - emptySet(), headers); - } - /** * Updates a document using the Update API. * See Update API on elastic.co @@ -573,17 +438,6 @@ public final UpdateResponse update(UpdateRequest updateRequest, RequestOptions o return performRequestAndParseEntity(updateRequest, RequestConverters::update, options, UpdateResponse::fromXContent, emptySet()); } - /** - * Updates a document using the Update API. - *

- * See Update API on elastic.co - * @deprecated Prefer {@link #update(UpdateRequest, RequestOptions)} - */ - @Deprecated - public final UpdateResponse update(UpdateRequest updateRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(updateRequest, RequestConverters::update, UpdateResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously updates a document using the Update API. * See Update API on elastic.co @@ -596,18 +450,6 @@ public final void updateAsync(UpdateRequest updateRequest, RequestOptions option emptySet()); } - /** - * Asynchronously updates a document using the Update API. - *

- * See Update API on elastic.co - * @deprecated Prefer {@link #updateAsync(UpdateRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void updateAsync(UpdateRequest updateRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(updateRequest, RequestConverters::update, UpdateResponse::fromXContent, listener, - emptySet(), headers); - } - /** * Deletes a document by id using the Delete API. * See Delete API on elastic.co @@ -621,18 +463,6 @@ public final DeleteResponse delete(DeleteRequest deleteRequest, RequestOptions o singleton(404)); } - /** - * Deletes a document by id using the Delete API. - * - * See Delete API on elastic.co - * @deprecated Prefer {@link #delete(DeleteRequest, RequestOptions)} - */ - @Deprecated - public final DeleteResponse delete(DeleteRequest deleteRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(deleteRequest, RequestConverters::delete, DeleteResponse::fromXContent, - singleton(404), headers); - } - /** * Asynchronously deletes a document by id using the Delete API. * See Delete API on elastic.co @@ -645,18 +475,6 @@ public final void deleteAsync(DeleteRequest deleteRequest, RequestOptions option Collections.singleton(404)); } - /** - * Asynchronously deletes a document by id using the Delete API. - * - * See Delete API on elastic.co - * @deprecated Prefer {@link #deleteAsync(DeleteRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void deleteAsync(DeleteRequest deleteRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(deleteRequest, RequestConverters::delete, DeleteResponse::fromXContent, listener, - Collections.singleton(404), headers); - } - /** * Executes a search request using the Search API. * See Search API on elastic.co @@ -669,17 +487,6 @@ public final SearchResponse search(SearchRequest searchRequest, RequestOptions o return performRequestAndParseEntity(searchRequest, RequestConverters::search, options, SearchResponse::fromXContent, emptySet()); } - /** - * Executes a search using the Search API. - * - * See Search API on elastic.co - * @deprecated Prefer {@link #search(SearchRequest, RequestOptions)} - */ - @Deprecated - public final SearchResponse search(SearchRequest searchRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(searchRequest, RequestConverters::search, SearchResponse::fromXContent, emptySet(), headers); - } - /** * Asynchronously executes a search using the Search API. * See Search API on elastic.co @@ -692,18 +499,6 @@ public final void searchAsync(SearchRequest searchRequest, RequestOptions option emptySet()); } - /** - * Asynchronously executes a search using the Search API. - * - * See Search API on elastic.co - * @deprecated Prefer {@link #searchAsync(SearchRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void searchAsync(SearchRequest searchRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(searchRequest, RequestConverters::search, SearchResponse::fromXContent, listener, - emptySet(), headers); - } - /** * Executes a multi search using the msearch API. * See Multi search API on @@ -718,19 +513,6 @@ public final MultiSearchResponse multiSearch(MultiSearchRequest multiSearchReque emptySet()); } - /** - * Executes a multi search using the msearch API. - * - * See Multi search API on - * elastic.co - * @deprecated Prefer {@link #multiSearch(MultiSearchRequest, RequestOptions)} - */ - @Deprecated - public final MultiSearchResponse multiSearch(MultiSearchRequest multiSearchRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(multiSearchRequest, RequestConverters::multiSearch, MultiSearchResponse::fromXContext, - emptySet(), headers); - } - /** * Asynchronously executes a multi search using the msearch API. * See Multi search API on @@ -745,19 +527,6 @@ public final void multiSearchAsync(MultiSearchRequest searchRequest, RequestOpti listener, emptySet()); } - /** - * Asynchronously executes a multi search using the msearch API. - * - * See Multi search API on - * elastic.co - * @deprecated Prefer {@link #multiSearchAsync(MultiSearchRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void multiSearchAsync(MultiSearchRequest searchRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(searchRequest, RequestConverters::multiSearch, MultiSearchResponse::fromXContext, listener, - emptySet(), headers); - } - /** * Executes a search using the Search Scroll API. * See Search Scroll @@ -772,19 +541,6 @@ public final SearchResponse searchScroll(SearchScrollRequest searchScrollRequest emptySet()); } - /** - * Executes a search using the Search Scroll API. - * - * See Search Scroll - * API on elastic.co - * @deprecated Prefer {@link #searchScroll(SearchScrollRequest, RequestOptions)} - */ - @Deprecated - public final SearchResponse searchScroll(SearchScrollRequest searchScrollRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, SearchResponse::fromXContent, - emptySet(), headers); - } - /** * Asynchronously executes a search using the Search Scroll API. * See Search Scroll @@ -799,20 +555,6 @@ public final void searchScrollAsync(SearchScrollRequest searchScrollRequest, Req listener, emptySet()); } - /** - * Asynchronously executes a search using the Search Scroll API. - * - * See Search Scroll - * API on elastic.co - * @deprecated Prefer {@link #searchScrollAsync(SearchScrollRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void searchScrollAsync(SearchScrollRequest searchScrollRequest, - ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(searchScrollRequest, RequestConverters::searchScroll, SearchResponse::fromXContent, - listener, emptySet(), headers); - } - /** * Clears one or more scroll ids using the Clear Scroll API. * See @@ -827,19 +569,6 @@ public final ClearScrollResponse clearScroll(ClearScrollRequest clearScrollReque emptySet()); } - /** - * Clears one or more scroll ids using the Clear Scroll API. - * - * See - * Clear Scroll API on elastic.co - * @deprecated Prefer {@link #clearScroll(ClearScrollRequest, RequestOptions)} - */ - @Deprecated - public final ClearScrollResponse clearScroll(ClearScrollRequest clearScrollRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(clearScrollRequest, RequestConverters::clearScroll, ClearScrollResponse::fromXContent, - emptySet(), headers); - } - /** * Asynchronously clears one or more scroll ids using the Clear Scroll API. * See @@ -854,20 +583,6 @@ public final void clearScrollAsync(ClearScrollRequest clearScrollRequest, Reques listener, emptySet()); } - /** - * Asynchronously clears one or more scroll ids using the Clear Scroll API. - * - * See - * Clear Scroll API on elastic.co - * @deprecated Prefer {@link #clearScrollAsync(ClearScrollRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void clearScrollAsync(ClearScrollRequest clearScrollRequest, - ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(clearScrollRequest, RequestConverters::clearScroll, ClearScrollResponse::fromXContent, - listener, emptySet(), headers); - } - /** * Executes a request using the Search Template API. * See Search Template API @@ -909,19 +624,6 @@ public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, RequestO emptySet()); } - /** - * Executes a request using the Ranking Evaluation API. - * - * See Ranking Evaluation API - * on elastic.co - * @deprecated Prefer {@link #rankEval(RankEvalRequest, RequestOptions)} - */ - @Deprecated - public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, Header... headers) throws IOException { - return performRequestAndParseEntity(rankEvalRequest, RequestConverters::rankEval, RankEvalResponse::fromXContent, - emptySet(), headers); - } - /** * Asynchronously executes a request using the Ranking Evaluation API. * See Ranking Evaluation API @@ -935,19 +637,6 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions emptySet()); } - /** - * Asynchronously executes a request using the Ranking Evaluation API. - * - * See Ranking Evaluation API - * on elastic.co - * @deprecated Prefer {@link #rankEvalAsync(RankEvalRequest, RequestOptions, ActionListener)} - */ - @Deprecated - public final void rankEvalAsync(RankEvalRequest rankEvalRequest, ActionListener listener, Header... headers) { - performRequestAsyncAndParseEntity(rankEvalRequest, RequestConverters::rankEval, RankEvalResponse::fromXContent, listener, - emptySet(), headers); - } - /** * Executes a request using the Field Capabilities API. * See Field Capabilities API @@ -977,14 +666,6 @@ public final void fieldCapsAsync(FieldCapabilitiesRequest fieldCapabilitiesReque FieldCapabilitiesResponse::fromXContent, listener, emptySet()); } - @Deprecated - protected final Resp performRequestAndParseEntity(Req request, - CheckedFunction requestConverter, - CheckedFunction entityParser, - Set ignores, Header... headers) throws IOException { - return performRequest(request, requestConverter, (response) -> parseEntity(response.getEntity(), entityParser), ignores, headers); - } - protected final Resp performRequestAndParseEntity(Req request, CheckedFunction requestConverter, RequestOptions options, @@ -994,14 +675,6 @@ protected final Resp performRequestAndParseEnt response -> parseEntity(response.getEntity(), entityParser), ignores); } - @Deprecated - protected final Resp performRequest(Req request, - CheckedFunction requestConverter, - CheckedFunction responseConverter, - Set ignores, Header... headers) throws IOException { - return performRequest(request, requestConverter, optionsForHeaders(headers), responseConverter, ignores); - } - protected final Resp performRequest(Req request, CheckedFunction requestConverter, RequestOptions options, @@ -1038,15 +711,6 @@ protected final Resp performRequest(Req reques } } - @Deprecated - protected final void performRequestAsyncAndParseEntity(Req request, - CheckedFunction requestConverter, - CheckedFunction entityParser, - ActionListener listener, Set ignores, Header... headers) { - performRequestAsync(request, requestConverter, (response) -> parseEntity(response.getEntity(), entityParser), - listener, ignores, headers); - } - protected final void performRequestAsyncAndParseEntity(Req request, CheckedFunction requestConverter, RequestOptions options, @@ -1056,14 +720,6 @@ protected final void performRequestAsyncAndPar response -> parseEntity(response.getEntity(), entityParser), listener, ignores); } - @Deprecated - protected final void performRequestAsync(Req request, - CheckedFunction requestConverter, - CheckedFunction responseConverter, - ActionListener listener, Set ignores, Header... headers) { - performRequestAsync(request, requestConverter, optionsForHeaders(headers), responseConverter, listener, ignores); - } - protected final void performRequestAsync(Req request, CheckedFunction requestConverter, RequestOptions options, diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java index f8f03d7f7d288..ebba636b8fa05 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/TasksClient.java @@ -84,7 +84,7 @@ public CancelTasksResponse cancel(CancelTasksRequest cancelTasksRequest, Request cancelTasksRequest, RequestConverters::cancelTasks, options, - parser -> CancelTasksResponse.fromXContent(parser), + CancelTasksResponse::fromXContent, emptySet() ); } @@ -103,7 +103,7 @@ public void cancelAsync(CancelTasksRequest cancelTasksRequest, RequestOptions op cancelTasksRequest, RequestConverters::cancelTasks, options, - parser -> CancelTasksResponse.fromXContent(parser), + CancelTasksResponse::fromXContent, listener, emptySet() ); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java index d41c47177f968..7605b1c715c74 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorIT.java @@ -56,7 +56,8 @@ public class BulkProcessorIT extends ESRestHighLevelClientTestCase { private static BulkProcessor.Builder initBulkProcessorBuilder(BulkProcessor.Listener listener) { - return BulkProcessor.builder(highLevelClient()::bulkAsync, listener); + return BulkProcessor.builder( + (request, bulkListener) -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); } public void testThatBulkProcessorCountIsCorrect() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java index fe6aa6b1017ee..c20998eeb5826 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/BulkProcessorRetryIT.java @@ -48,7 +48,8 @@ public class BulkProcessorRetryIT extends ESRestHighLevelClientTestCase { private static final String TYPE_NAME = "type"; private static BulkProcessor.Builder initBulkProcessorBuilder(BulkProcessor.Listener listener) { - return BulkProcessor.builder(highLevelClient()::bulkAsync, listener); + return BulkProcessor.builder( + (request, bulkListener) -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); } public void testBulkRejectionLoadWithoutBackoff() throws Exception { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java index 7cf9fca07c30d..2a870ec65eea1 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java @@ -63,7 +63,6 @@ public void testClusterPutSettings() throws IOException { setRequest.persistentSettings(map); ClusterUpdateSettingsResponse setResponse = execute(setRequest, highLevelClient().cluster()::putSettings, - highLevelClient().cluster()::putSettingsAsync, highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync); assertAcked(setResponse); @@ -86,7 +85,6 @@ public void testClusterPutSettings() throws IOException { resetRequest.persistentSettings("{\"" + persistentSettingKey + "\": null }", XContentType.JSON); ClusterUpdateSettingsResponse resetResponse = execute(resetRequest, highLevelClient().cluster()::putSettings, - highLevelClient().cluster()::putSettingsAsync, highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync); assertThat(resetResponse.getTransientSettings().get(transientSettingKey), equalTo(null)); @@ -108,7 +106,6 @@ public void testClusterUpdateSettingNonExistent() { clusterUpdateSettingsRequest.transientSettings(Settings.builder().put(setting, value).build()); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(clusterUpdateSettingsRequest, - highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync, highLevelClient().cluster()::putSettings, highLevelClient().cluster()::putSettingsAsync)); assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(exception.getMessage(), equalTo( @@ -132,7 +129,6 @@ public void testClusterHealthYellowClusterLevel() throws IOException { createIndex("index2", Settings.EMPTY); ClusterHealthRequest request = new ClusterHealthRequest(); request.timeout("5s"); - request.level(ClusterHealthRequest.Level.CLUSTER); ClusterHealthResponse response = execute(request, highLevelClient().cluster()::health, highLevelClient().cluster()::healthAsync); assertYellowShards(response); @@ -173,6 +169,7 @@ public void testClusterHealthYellowSpecificIndex() throws IOException { createIndex("index", Settings.EMPTY); createIndex("index2", Settings.EMPTY); ClusterHealthRequest request = new ClusterHealthRequest("index"); + request.level(ClusterHealthRequest.Level.SHARDS); request.timeout("5s"); ClusterHealthResponse response = execute(request, highLevelClient().cluster()::health, highLevelClient().cluster()::healthAsync); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java index 81c894f242fe4..9de4c22611c3b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/CrudIT.java @@ -74,8 +74,7 @@ public void testDelete() throws IOException { if (randomBoolean()) { deleteRequest.version(1L); } - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, - highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -85,8 +84,7 @@ public void testDelete() throws IOException { // Testing non existing document String docId = "does_not_exist"; DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId); - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, - highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -99,8 +97,7 @@ public void testDelete() throws IOException { new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).version(2); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, - highLevelClient()::delete, highLevelClient()::deleteAsync)); + () -> execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + docId + "]: " + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); @@ -113,8 +110,7 @@ public void testDelete() throws IOException { new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")) .versionType(VersionType.EXTERNAL).version(12), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).versionType(VersionType.EXTERNAL).version(13); - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, - highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -128,8 +124,7 @@ public void testDelete() throws IOException { .versionType(VersionType.EXTERNAL).version(12), RequestOptions.DEFAULT); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).versionType(VersionType.EXTERNAL).version(10); - execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, - highLevelClient()::delete, highLevelClient()::deleteAsync); + execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][" + @@ -142,8 +137,7 @@ public void testDelete() throws IOException { highLevelClient().index(new IndexRequest("index", "type", docId).source(Collections.singletonMap("foo", "bar")).routing("foo"), RequestOptions.DEFAULT); DeleteRequest deleteRequest = new DeleteRequest("index", "type", docId).routing("foo"); - DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync, - highLevelClient()::delete, highLevelClient()::deleteAsync); + DeleteResponse deleteResponse = execute(deleteRequest, highLevelClient()::delete, highLevelClient()::deleteAsync); assertEquals("index", deleteResponse.getIndex()); assertEquals("type", deleteResponse.getType()); assertEquals(docId, deleteResponse.getId()); @@ -154,8 +148,7 @@ public void testDelete() throws IOException { public void testExists() throws IOException { { GetRequest getRequest = new GetRequest("index", "type", "id"); - assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, - highLevelClient()::exists, highLevelClient()::existsAsync)); + assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); } IndexRequest index = new IndexRequest("index", "type", "id"); index.source("{\"field1\":\"value1\",\"field2\":\"value2\"}", XContentType.JSON); @@ -163,18 +156,15 @@ public void testExists() throws IOException { highLevelClient().index(index, RequestOptions.DEFAULT); { GetRequest getRequest = new GetRequest("index", "type", "id"); - assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, - highLevelClient()::exists, highLevelClient()::existsAsync)); + assertTrue(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); } { GetRequest getRequest = new GetRequest("index", "type", "does_not_exist"); - assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, - highLevelClient()::exists, highLevelClient()::existsAsync)); + assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); } { GetRequest getRequest = new GetRequest("index", "type", "does_not_exist").version(1); - assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync, - highLevelClient()::exists, highLevelClient()::existsAsync)); + assertFalse(execute(getRequest, highLevelClient()::exists, highLevelClient()::existsAsync)); } } @@ -182,8 +172,7 @@ public void testGet() throws IOException { { GetRequest getRequest = new GetRequest("index", "type", "id"); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, - highLevelClient()::get, highLevelClient()::getAsync)); + () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertEquals("Elasticsearch exception [type=index_not_found_exception, reason=no such index]", exception.getMessage()); assertEquals("index", exception.getMetadata("es.index").get(0)); @@ -196,8 +185,7 @@ public void testGet() throws IOException { { GetRequest getRequest = new GetRequest("index", "type", "id").version(2); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, - highLevelClient()::get, highLevelClient()::getAsync)); + () -> execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, " + "reason=[type][id]: " + "version conflict, current version [1] is different than the one provided [2]]", exception.getMessage()); @@ -208,8 +196,7 @@ public void testGet() throws IOException { if (randomBoolean()) { getRequest.version(1L); } - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, - highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("id", getResponse.getId()); @@ -220,8 +207,7 @@ public void testGet() throws IOException { } { GetRequest getRequest = new GetRequest("index", "type", "does_not_exist"); - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, - highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("does_not_exist", getResponse.getId()); @@ -233,8 +219,7 @@ public void testGet() throws IOException { { GetRequest getRequest = new GetRequest("index", "type", "id"); getRequest.fetchSourceContext(new FetchSourceContext(false, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY)); - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, - highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("id", getResponse.getId()); @@ -250,8 +235,7 @@ public void testGet() throws IOException { } else { getRequest.fetchSourceContext(new FetchSourceContext(true, Strings.EMPTY_ARRAY, new String[]{"field2"})); } - GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync, - highLevelClient()::get, highLevelClient()::getAsync); + GetResponse getResponse = execute(getRequest, highLevelClient()::get, highLevelClient()::getAsync); assertEquals("index", getResponse.getIndex()); assertEquals("type", getResponse.getType()); assertEquals("id", getResponse.getId()); @@ -269,8 +253,7 @@ public void testMultiGet() throws IOException { MultiGetRequest multiGetRequest = new MultiGetRequest(); multiGetRequest.add("index", "type", "id1"); multiGetRequest.add("index", "type", "id2"); - MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync, - highLevelClient()::multiGet, highLevelClient()::multiGetAsync); + MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync); assertEquals(2, response.getResponses().length); assertTrue(response.getResponses()[0].isFailed()); @@ -302,8 +285,7 @@ public void testMultiGet() throws IOException { MultiGetRequest multiGetRequest = new MultiGetRequest(); multiGetRequest.add("index", "type", "id1"); multiGetRequest.add("index", "type", "id2"); - MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync, - highLevelClient()::multiGet, highLevelClient()::multiGetAsync); + MultiGetResponse response = execute(multiGetRequest, highLevelClient()::multiGet, highLevelClient()::multiGetAsync); assertEquals(2, response.getResponses().length); assertFalse(response.getResponses()[0].isFailed()); @@ -328,8 +310,7 @@ public void testIndex() throws IOException { IndexRequest indexRequest = new IndexRequest("index", "type"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("test", "test").endObject()); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult()); assertEquals("index", indexResponse.getIndex()); @@ -350,8 +331,7 @@ public void testIndex() throws IOException { IndexRequest indexRequest = new IndexRequest("index", "type", "id"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("version", 1).endObject()); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); @@ -361,8 +341,7 @@ public void testIndex() throws IOException { indexRequest = new IndexRequest("index", "type", "id"); indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("version", 2).endObject()); - indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.OK, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); @@ -374,8 +353,7 @@ public void testIndex() throws IOException { wrongRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); wrongRequest.version(5L); - execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + execute(wrongRequest, highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: " + @@ -388,8 +366,7 @@ public void testIndex() throws IOException { indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); indexRequest.setPipeline("missing"); - execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.BAD_REQUEST, exception.status()); @@ -402,8 +379,7 @@ public void testIndex() throws IOException { indexRequest.version(12L); indexRequest.versionType(VersionType.EXTERNAL); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); @@ -415,16 +391,14 @@ public void testIndex() throws IOException { indexRequest.source(XContentBuilder.builder(xContentType.xContent()).startObject().field("field", "test").endObject()); indexRequest.opType(DocWriteRequest.OpType.CREATE); - IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + IndexResponse indexResponse = execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); assertEquals(RestStatus.CREATED, indexResponse.status()); assertEquals("index", indexResponse.getIndex()); assertEquals("type", indexResponse.getType()); assertEquals("with_create_op_type", indexResponse.getId()); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> { - execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync, - highLevelClient()::index, highLevelClient()::indexAsync); + execute(indexRequest, highLevelClient()::index, highLevelClient()::indexAsync); }); assertEquals(RestStatus.CONFLICT, exception.status()); @@ -439,8 +413,7 @@ public void testUpdate() throws IOException { updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values())); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> - execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync)); + execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertEquals("Elasticsearch exception [type=document_missing_exception, reason=[type][does_not_exist]: document missing]", exception.getMessage()); @@ -463,8 +436,7 @@ public void testUpdate() throws IOException { updateRequestConflict.version(indexResponse.getVersion()); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> - execute(updateRequestConflict, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync)); + execute(updateRequestConflict, highLevelClient()::update, highLevelClient()::updateAsync)); assertEquals(RestStatus.CONFLICT, exception.status()); assertEquals("Elasticsearch exception [type=version_conflict_engine_exception, reason=[type][id]: version conflict, " + "current version [2] is different than the one provided [1]]", exception.getMessage()); @@ -480,8 +452,7 @@ public void testUpdate() throws IOException { updateRequest.script(script); updateRequest.fetchSource(true); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.OK, updateResponse.status()); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); assertEquals(2L, updateResponse.getVersion()); @@ -501,8 +472,7 @@ public void testUpdate() throws IOException { updateRequest.doc(singletonMap("field_2", "two"), randomFrom(XContentType.values())); updateRequest.fetchSource("field_*", "field_3"); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.OK, updateResponse.status()); assertEquals(DocWriteResponse.Result.UPDATED, updateResponse.getResult()); assertEquals(13L, updateResponse.getVersion()); @@ -523,8 +493,7 @@ public void testUpdate() throws IOException { UpdateRequest updateRequest = new UpdateRequest("index", "type", "noop"); updateRequest.doc(singletonMap("field", "value"), randomFrom(XContentType.values())); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.OK, updateResponse.status()); assertEquals(DocWriteResponse.Result.NOOP, updateResponse.getResult()); assertEquals(1L, updateResponse.getVersion()); @@ -542,8 +511,7 @@ public void testUpdate() throws IOException { updateRequest.doc(singletonMap("doc_status", "updated")); updateRequest.fetchSource(true); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.CREATED, updateResponse.status()); assertEquals("index", updateResponse.getIndex()); assertEquals("type", updateResponse.getType()); @@ -558,8 +526,7 @@ public void testUpdate() throws IOException { updateRequest.fetchSource(true); updateRequest.docAsUpsert(true); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.CREATED, updateResponse.status()); assertEquals("index", updateResponse.getIndex()); assertEquals("type", updateResponse.getType()); @@ -575,8 +542,7 @@ public void testUpdate() throws IOException { updateRequest.scriptedUpsert(true); updateRequest.upsert(singletonMap("level", "A")); - UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync); + UpdateResponse updateResponse = execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); assertEquals(RestStatus.CREATED, updateResponse.status()); assertEquals("index", updateResponse.getIndex()); assertEquals("type", updateResponse.getType()); @@ -591,8 +557,7 @@ public void testUpdate() throws IOException { UpdateRequest updateRequest = new UpdateRequest("index", "type", "id"); updateRequest.doc(new IndexRequest().source(Collections.singletonMap("field", "doc"), XContentType.JSON)); updateRequest.upsert(new IndexRequest().source(Collections.singletonMap("field", "upsert"), XContentType.YAML)); - execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync, - highLevelClient()::update, highLevelClient()::updateAsync); + execute(updateRequest, highLevelClient()::update, highLevelClient()::updateAsync); }); assertEquals("Update request cannot have different content types for doc [JSON] and upsert [YAML] documents", exception.getMessage()); @@ -651,8 +616,7 @@ public void testBulk() throws IOException { } } - BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync, - highLevelClient()::bulk, highLevelClient()::bulkAsync); + BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync); assertEquals(RestStatus.OK, bulkResponse.status()); assertTrue(bulkResponse.getTook().getMillis() > 0); assertEquals(nbItems, bulkResponse.getItems().length); @@ -688,10 +652,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) } }; - // Pull the client to a variable to work around https://bugs.eclipse.org/bugs/show_bug.cgi?id=514884 - RestHighLevelClient hlClient = highLevelClient(); - - try (BulkProcessor processor = BulkProcessor.builder(hlClient::bulkAsync, listener) + try (BulkProcessor processor = BulkProcessor.builder( + (request, bulkListener) -> highLevelClient().bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener) .setConcurrentRequests(0) .setBulkSize(new ByteSizeValue(5, ByteSizeUnit.GB)) .setBulkActions(nbItems + 1) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java index 14fe0e01d31f9..4ad39f547584b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/ESRestHighLevelClientTestCase.java @@ -19,7 +19,6 @@ package org.elasticsearch.client; -import org.apache.http.Header; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ingest.PutPipelineRequest; import org.elasticsearch.action.support.PlainActionFuture; @@ -80,43 +79,6 @@ protected interface AsyncMethod { void execute(Request request, RequestOptions options, ActionListener listener); } - /** - * Executes the provided request using either the sync method or its async variant, both provided as functions - */ - @Deprecated - protected static Resp execute(Req request, SyncMethod syncMethod, AsyncMethod asyncMethod, - SyncMethodWithHeaders syncMethodWithHeaders, - AsyncMethodWithHeaders asyncMethodWithHeaders) throws IOException { - switch(randomIntBetween(0, 3)) { - case 0: - return syncMethod.execute(request, RequestOptions.DEFAULT); - case 1: - PlainActionFuture future = PlainActionFuture.newFuture(); - asyncMethod.execute(request, RequestOptions.DEFAULT, future); - return future.actionGet(); - case 2: - return syncMethodWithHeaders.execute(request); - case 3: - PlainActionFuture futureWithHeaders = PlainActionFuture.newFuture(); - asyncMethodWithHeaders.execute(request, futureWithHeaders); - return futureWithHeaders.actionGet(); - default: - throw new UnsupportedOperationException(); - } - } - - @Deprecated - @FunctionalInterface - protected interface SyncMethodWithHeaders { - Response execute(Request request, Header... headers) throws IOException; - } - - @Deprecated - @FunctionalInterface - protected interface AsyncMethodWithHeaders { - void execute(Request request, ActionListener listener, Header... headers); - } - private static class HighLevelClient extends RestHighLevelClient { private HighLevelClient(RestClient restClient) { super(restClient, (client) -> {}, Collections.emptyList()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 9b54938ede4d6..82ac161f5afe0 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -117,8 +117,6 @@ public void testIndicesExists() throws IOException { boolean response = execute( request, highLevelClient().indices()::exists, - highLevelClient().indices()::existsAsync, - highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertTrue(response); @@ -134,8 +132,6 @@ public void testIndicesExists() throws IOException { boolean response = execute( request, highLevelClient().indices()::exists, - highLevelClient().indices()::existsAsync, - highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertFalse(response); @@ -154,8 +150,6 @@ public void testIndicesExists() throws IOException { boolean response = execute( request, highLevelClient().indices()::exists, - highLevelClient().indices()::existsAsync, - highLevelClient().indices()::exists, highLevelClient().indices()::existsAsync ); assertFalse(response); @@ -173,8 +167,7 @@ public void testCreateIndex() throws IOException { CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName); CreateIndexResponse createIndexResponse = - execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync, - highLevelClient().indices()::create, highLevelClient().indices()::createAsync); + execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync); assertTrue(createIndexResponse.isAcknowledged()); assertTrue(indexExists(indexName)); @@ -202,8 +195,7 @@ public void testCreateIndex() throws IOException { createIndexRequest.mapping("type_name", mappingBuilder); CreateIndexResponse createIndexResponse = - execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync, - highLevelClient().indices()::create, highLevelClient().indices()::createAsync); + execute(createIndexRequest, highLevelClient().indices()::create, highLevelClient().indices()::createAsync); assertTrue(createIndexResponse.isAcknowledged()); Map getIndexResponse = getAsMap(indexName); @@ -338,8 +330,7 @@ public void testPutMapping() throws IOException { putMappingRequest.source(mappingBuilder); PutMappingResponse putMappingResponse = - execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync, - highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync); + execute(putMappingRequest, highLevelClient().indices()::putMapping, highLevelClient().indices()::putMappingAsync); assertTrue(putMappingResponse.isAcknowledged()); Map getIndexResponse = getAsMap(indexName); @@ -390,8 +381,7 @@ public void testDeleteIndex() throws IOException { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(indexName); DeleteIndexResponse deleteIndexResponse = - execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync, - highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync); + execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync); assertTrue(deleteIndexResponse.isAcknowledged()); assertFalse(indexExists(indexName)); @@ -404,8 +394,7 @@ public void testDeleteIndex() throws IOException { DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync, - highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync)); + () -> execute(deleteIndexRequest, highLevelClient().indices()::delete, highLevelClient().indices()::deleteAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -424,7 +413,6 @@ public void testUpdateAliases() throws IOException { addAction.routing("routing").searchRouting("search_routing").filter("{\"term\":{\"year\":2016}}"); aliasesAddRequest.addAliasAction(addAction); IndicesAliasesResponse aliasesAddResponse = execute(aliasesAddRequest, highLevelClient().indices()::updateAliases, - highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync); assertTrue(aliasesAddResponse.isAcknowledged()); assertThat(aliasExists(alias), equalTo(true)); @@ -443,7 +431,6 @@ public void testUpdateAliases() throws IOException { AliasActions removeAction = new AliasActions(AliasActions.Type.REMOVE).index(index).alias(alias); aliasesAddRemoveRequest.addAliasAction(removeAction); IndicesAliasesResponse aliasesAddRemoveResponse = execute(aliasesAddRemoveRequest, highLevelClient().indices()::updateAliases, - highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync); assertTrue(aliasesAddRemoveResponse.isAcknowledged()); assertThat(aliasExists(alias), equalTo(false)); @@ -455,7 +442,6 @@ public void testUpdateAliases() throws IOException { AliasActions removeIndexAction = new AliasActions(AliasActions.Type.REMOVE_INDEX).index(index); aliasesRemoveIndexRequest.addAliasAction(removeIndexAction); IndicesAliasesResponse aliasesRemoveIndexResponse = execute(aliasesRemoveIndexRequest, highLevelClient().indices()::updateAliases, - highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync); assertTrue(aliasesRemoveIndexResponse.isAcknowledged()); assertThat(aliasExists(alias), equalTo(false)); @@ -473,9 +459,7 @@ public void testAliasesNonExistentIndex() throws IOException { IndicesAliasesRequest nonExistentIndexRequest = new IndicesAliasesRequest(); nonExistentIndexRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index(nonExistentIndex).alias(alias)); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(nonExistentIndexRequest, - highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync, - highLevelClient().indices()::updateAliases, - highLevelClient().indices()::updateAliasesAsync)); + highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND)); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); assertThat(exception.getMetadata("es.index"), hasItem(nonExistentIndex)); @@ -485,8 +469,7 @@ public void testAliasesNonExistentIndex() throws IOException { mixedRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).indices(index).aliases(alias)); mixedRequest.addAliasAction(new AliasActions(AliasActions.Type.REMOVE).indices(nonExistentIndex).alias(alias)); exception = expectThrows(ElasticsearchStatusException.class, - () -> execute(mixedRequest, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync, - highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); + () -> execute(mixedRequest, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND)); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); assertThat(exception.getMetadata("es.index"), hasItem(nonExistentIndex)); @@ -498,7 +481,6 @@ public void testAliasesNonExistentIndex() throws IOException { removeIndexRequest.addAliasAction(new AliasActions(AliasActions.Type.ADD).index(nonExistentIndex).alias(alias)); removeIndexRequest.addAliasAction(new AliasActions(AliasActions.Type.REMOVE_INDEX).indices(nonExistentIndex)); exception = expectThrows(ElasticsearchException.class, () -> execute(removeIndexRequest, highLevelClient().indices()::updateAliases, - highLevelClient().indices()::updateAliasesAsync, highLevelClient().indices()::updateAliases, highLevelClient().indices()::updateAliasesAsync)); assertThat(exception.status(), equalTo(RestStatus.NOT_FOUND)); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); @@ -519,7 +501,6 @@ public void testOpenExistingIndex() throws IOException { OpenIndexRequest openIndexRequest = new OpenIndexRequest(index); OpenIndexResponse openIndexResponse = execute(openIndexRequest, highLevelClient().indices()::open, - highLevelClient().indices()::openAsync, highLevelClient().indices()::open, highLevelClient().indices()::openAsync); assertTrue(openIndexResponse.isAcknowledged()); @@ -533,22 +514,19 @@ public void testOpenNonExistentIndex() throws IOException { OpenIndexRequest openIndexRequest = new OpenIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync, - highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); + () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); OpenIndexRequest lenientOpenIndexRequest = new OpenIndexRequest(nonExistentIndex); lenientOpenIndexRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); OpenIndexResponse lenientOpenIndexResponse = execute(lenientOpenIndexRequest, highLevelClient().indices()::open, - highLevelClient().indices()::openAsync, highLevelClient().indices()::open, highLevelClient().indices()::openAsync); assertThat(lenientOpenIndexResponse.isAcknowledged(), equalTo(true)); OpenIndexRequest strictOpenIndexRequest = new OpenIndexRequest(nonExistentIndex); strictOpenIndexRequest.indicesOptions(IndicesOptions.strictExpandOpen()); ElasticsearchException strictException = expectThrows(ElasticsearchException.class, - () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync, - highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); + () -> execute(openIndexRequest, highLevelClient().indices()::open, highLevelClient().indices()::openAsync)); assertEquals(RestStatus.NOT_FOUND, strictException.status()); } @@ -560,7 +538,6 @@ public void testCloseExistingIndex() throws IOException { CloseIndexRequest closeIndexRequest = new CloseIndexRequest(index); CloseIndexResponse closeIndexResponse = execute(closeIndexRequest, highLevelClient().indices()::close, - highLevelClient().indices()::closeAsync, highLevelClient().indices()::close, highLevelClient().indices()::closeAsync); assertTrue(closeIndexResponse.isAcknowledged()); @@ -576,8 +553,7 @@ public void testCloseNonExistentIndex() throws IOException { CloseIndexRequest closeIndexRequest = new CloseIndexRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(closeIndexRequest, highLevelClient().indices()::close, highLevelClient().indices()::closeAsync, - highLevelClient().indices()::close, highLevelClient().indices()::closeAsync)); + () -> execute(closeIndexRequest, highLevelClient().indices()::close, highLevelClient().indices()::closeAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } @@ -591,8 +567,7 @@ public void testRefresh() throws IOException { createIndex(index, settings); RefreshRequest refreshRequest = new RefreshRequest(index); RefreshResponse refreshResponse = - execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync, - highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync); + execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync); assertThat(refreshResponse.getTotalShards(), equalTo(1)); assertThat(refreshResponse.getSuccessfulShards(), equalTo(1)); assertThat(refreshResponse.getFailedShards(), equalTo(0)); @@ -603,8 +578,7 @@ public void testRefresh() throws IOException { assertFalse(indexExists(nonExistentIndex)); RefreshRequest refreshRequest = new RefreshRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync, - highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync)); + () -> execute(refreshRequest, highLevelClient().indices()::refresh, highLevelClient().indices()::refreshAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -619,8 +593,7 @@ public void testFlush() throws IOException { createIndex(index, settings); FlushRequest flushRequest = new FlushRequest(index); FlushResponse flushResponse = - execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync, - highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync); + execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync); assertThat(flushResponse.getTotalShards(), equalTo(1)); assertThat(flushResponse.getSuccessfulShards(), equalTo(1)); assertThat(flushResponse.getFailedShards(), equalTo(0)); @@ -631,8 +604,7 @@ public void testFlush() throws IOException { assertFalse(indexExists(nonExistentIndex)); FlushRequest flushRequest = new FlushRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync, - highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync)); + () -> execute(flushRequest, highLevelClient().indices()::flush, highLevelClient().indices()::flushAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -680,8 +652,7 @@ public void testClearCache() throws IOException { createIndex(index, settings); ClearIndicesCacheRequest clearCacheRequest = new ClearIndicesCacheRequest(index); ClearIndicesCacheResponse clearCacheResponse = - execute(clearCacheRequest, highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync, - highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync); + execute(clearCacheRequest, highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync); assertThat(clearCacheResponse.getTotalShards(), equalTo(1)); assertThat(clearCacheResponse.getSuccessfulShards(), equalTo(1)); assertThat(clearCacheResponse.getFailedShards(), equalTo(0)); @@ -692,8 +663,8 @@ public void testClearCache() throws IOException { assertFalse(indexExists(nonExistentIndex)); ClearIndicesCacheRequest clearCacheRequest = new ClearIndicesCacheRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(clearCacheRequest, highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync, - highLevelClient().indices()::clearCache, highLevelClient().indices()::clearCacheAsync)); + () -> execute(clearCacheRequest, highLevelClient().indices()::clearCache, + highLevelClient().indices()::clearCacheAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } @@ -708,8 +679,7 @@ public void testForceMerge() throws IOException { createIndex(index, settings); ForceMergeRequest forceMergeRequest = new ForceMergeRequest(index); ForceMergeResponse forceMergeResponse = - execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync, - highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync); + execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync); assertThat(forceMergeResponse.getTotalShards(), equalTo(1)); assertThat(forceMergeResponse.getSuccessfulShards(), equalTo(1)); assertThat(forceMergeResponse.getFailedShards(), equalTo(0)); @@ -720,30 +690,25 @@ public void testForceMerge() throws IOException { assertFalse(indexExists(nonExistentIndex)); ForceMergeRequest forceMergeRequest = new ForceMergeRequest(nonExistentIndex); ElasticsearchException exception = expectThrows(ElasticsearchException.class, - () -> execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync, - highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync)); + () -> execute(forceMergeRequest, highLevelClient().indices()::forceMerge, highLevelClient().indices()::forceMergeAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); } } public void testExistsAlias() throws IOException { GetAliasesRequest getAliasesRequest = new GetAliasesRequest("alias"); - assertFalse(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, - highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertFalse(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); createIndex("index", Settings.EMPTY); client().performRequest(HttpPut.METHOD_NAME, "/index/_alias/alias"); - assertTrue(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, - highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertTrue(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); GetAliasesRequest getAliasesRequest2 = new GetAliasesRequest(); getAliasesRequest2.aliases("alias"); getAliasesRequest2.indices("index"); - assertTrue(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, - highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertTrue(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); getAliasesRequest2.indices("does_not_exist"); - assertFalse(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, - highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); + assertFalse(execute(getAliasesRequest2, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); } @SuppressWarnings("unchecked") @@ -764,7 +729,7 @@ public void testShrink() throws IOException { .build(); resizeRequest.setTargetIndex(new CreateIndexRequest("target").settings(targetSettings).alias(new Alias("alias"))); ResizeResponse resizeResponse = execute(resizeRequest, highLevelClient().indices()::shrink, - highLevelClient().indices()::shrinkAsync, highLevelClient().indices()::shrink, highLevelClient().indices()::shrinkAsync); + highLevelClient().indices()::shrinkAsync); assertTrue(resizeResponse.isAcknowledged()); assertTrue(resizeResponse.isShardsAcknowledged()); Map getIndexResponse = getAsMap("target"); @@ -786,8 +751,7 @@ public void testSplit() throws IOException { resizeRequest.setResizeType(ResizeType.SPLIT); Settings targetSettings = Settings.builder().put("index.number_of_shards", 4).put("index.number_of_replicas", 0).build(); resizeRequest.setTargetIndex(new CreateIndexRequest("target").settings(targetSettings).alias(new Alias("alias"))); - ResizeResponse resizeResponse = execute(resizeRequest, highLevelClient().indices()::split, highLevelClient().indices()::splitAsync, - highLevelClient().indices()::split, highLevelClient().indices()::splitAsync); + ResizeResponse resizeResponse = execute(resizeRequest, highLevelClient().indices()::split, highLevelClient().indices()::splitAsync); assertTrue(resizeResponse.isAcknowledged()); assertTrue(resizeResponse.isShardsAcknowledged()); Map getIndexResponse = getAsMap("target"); @@ -806,7 +770,6 @@ public void testRollover() throws IOException { { RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -826,7 +789,6 @@ public void testRollover() throws IOException { rolloverRequest.addMaxIndexAgeCondition(new TimeValue(1)); rolloverRequest.dryRun(true); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertFalse(rolloverResponse.isRolledOver()); assertTrue(rolloverResponse.isDryRun()); @@ -841,7 +803,6 @@ public void testRollover() throws IOException { rolloverRequest.dryRun(false); rolloverRequest.addMaxIndexSizeCondition(new ByteSizeValue(1, ByteSizeUnit.MB)); RolloverResponse rolloverResponse = execute(rolloverRequest, highLevelClient().indices()::rollover, - highLevelClient().indices()::rolloverAsync, highLevelClient().indices()::rollover, highLevelClient().indices()::rolloverAsync); assertTrue(rolloverResponse.isRolledOver()); assertFalse(rolloverResponse.isDryRun()); @@ -1067,7 +1028,6 @@ public void testIndexPutSettings() throws IOException { UpdateSettingsRequest dynamicSettingRequest = new UpdateSettingsRequest(); dynamicSettingRequest.settings(Settings.builder().put(dynamicSettingKey, dynamicSettingValue).build()); UpdateSettingsResponse response = execute(dynamicSettingRequest, highLevelClient().indices()::putSettings, - highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync); assertTrue(response.isAcknowledged()); @@ -1078,7 +1038,6 @@ public void testIndexPutSettings() throws IOException { UpdateSettingsRequest staticSettingRequest = new UpdateSettingsRequest(); staticSettingRequest.settings(Settings.builder().put(staticSettingKey, staticSettingValue).build()); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(staticSettingRequest, - highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertThat(exception.getMessage(), startsWith("Elasticsearch exception [type=illegal_argument_exception, " @@ -1089,7 +1048,6 @@ public void testIndexPutSettings() throws IOException { closeIndex(index); response = execute(staticSettingRequest, highLevelClient().indices()::putSettings, - highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync); assertTrue(response.isAcknowledged()); openIndex(index); @@ -1100,7 +1058,6 @@ public void testIndexPutSettings() throws IOException { UpdateSettingsRequest unmodifiableSettingRequest = new UpdateSettingsRequest(); unmodifiableSettingRequest.settings(Settings.builder().put(unmodifiableSettingKey, unmodifiableSettingValue).build()); exception = expectThrows(ElasticsearchException.class, () -> execute(unmodifiableSettingRequest, - highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertThat(exception.getMessage(), startsWith( "Elasticsearch exception [type=illegal_argument_exception, " @@ -1128,14 +1085,12 @@ public void testIndexPutSettingNonExistent() throws IOException { indexUpdateSettingsRequest.settings(Settings.builder().put(setting, value).build()); ElasticsearchException exception = expectThrows(ElasticsearchException.class, () -> execute(indexUpdateSettingsRequest, - highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertThat(exception.getMessage(), equalTo("Elasticsearch exception [type=index_not_found_exception, reason=no such index]")); createIndex(index, Settings.EMPTY); exception = expectThrows(ElasticsearchException.class, () -> execute(indexUpdateSettingsRequest, - highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync, highLevelClient().indices()::putSettings, highLevelClient().indices()::putSettingsAsync)); assertThat(exception.status(), equalTo(RestStatus.BAD_REQUEST)); assertThat(exception.getMessage(), equalTo( diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java index 1e12f3f5e62f6..a7a452484e023 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java @@ -82,8 +82,7 @@ public void testRankEvalRequest() throws IOException { RankEvalSpec spec = new RankEvalSpec(specifications, metric); RankEvalRequest rankEvalRequest = new RankEvalRequest(spec, new String[] { "index", "index2" }); - RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync, - highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); + RankEvalResponse response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); // the expected Prec@ for the first query is 5/7 and the expected Prec@ for the second is 1/7, divided by 2 to get the average double expectedPrecision = (1.0 / 7.0 + 5.0 / 7.0) / 2.0; assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE); @@ -117,8 +116,7 @@ public void testRankEvalRequest() throws IOException { // now try this when test2 is closed client().performRequest("POST", "index2/_close", Collections.emptyMap()); rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); - response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync, - highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); + response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); } private static List createRelevant(String indexName, String... docs) { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 1e03e55f61f13..aa8221f30991e 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1562,13 +1562,13 @@ public void testClusterHealth() { default: throw new UnsupportedOperationException(); } - setRandomWaitForActiveShards(healthRequest::waitForActiveShards, expectedParams, "0"); + setRandomWaitForActiveShards(healthRequest::waitForActiveShards, ActiveShardCount.NONE, expectedParams); if (randomBoolean()) { ClusterHealthRequest.Level level = randomFrom(ClusterHealthRequest.Level.values()); healthRequest.level(level); expectedParams.put("level", level.name().toLowerCase(Locale.ROOT)); } else { - expectedParams.put("level", "shards"); + expectedParams.put("level", "cluster"); } if (randomBoolean()) { Priority priority = randomFrom(Priority.values()); @@ -2187,23 +2187,24 @@ private static void setRandomMasterTimeout(MasterNodeRequest request, Map setter, Map expectedParams) { - setRandomWaitForActiveShards(setter, expectedParams, null); + setRandomWaitForActiveShards(setter, ActiveShardCount.DEFAULT, expectedParams); } - private static void setRandomWaitForActiveShards(Consumer setter,Map expectedParams, - String defaultValue) { + private static void setRandomWaitForActiveShards(Consumer setter, ActiveShardCount defaultActiveShardCount, + Map expectedParams) { if (randomBoolean()) { + int waitForActiveShardsInt = randomIntBetween(-1, 5); String waitForActiveShardsString; - int waitForActiveShards = randomIntBetween(-1, 5); - if (waitForActiveShards == -1) { + if (waitForActiveShardsInt == -1) { waitForActiveShardsString = "all"; } else { - waitForActiveShardsString = String.valueOf(waitForActiveShards); + waitForActiveShardsString = String.valueOf(waitForActiveShardsInt); + } + ActiveShardCount activeShardCount = ActiveShardCount.parseString(waitForActiveShardsString); + setter.accept(activeShardCount); + if (defaultActiveShardCount.equals(activeShardCount) == false) { + expectedParams.put("wait_for_active_shards", waitForActiveShardsString); } - setter.accept(ActiveShardCount.parseString(waitForActiveShardsString)); - expectedParams.put("wait_for_active_shards", waitForActiveShardsString); - } else if (defaultValue != null) { - expectedParams.put("wait_for_active_shards", defaultValue); } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index 9084a547c162d..49a84146dc8f4 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -188,12 +188,12 @@ public ActionRequestValidationException validate() { { ActionRequestValidationException actualException = expectThrows(ActionRequestValidationException.class, - () -> restHighLevelClient.performRequest(request, null, null, null)); + () -> restHighLevelClient.performRequest(request, null, RequestOptions.DEFAULT, null, null)); assertSame(validationException, actualException); } { TrackingActionListener trackingActionListener = new TrackingActionListener(); - restHighLevelClient.performRequestAsync(request, null, null, trackingActionListener, null); + restHighLevelClient.performRequestAsync(request, null, RequestOptions.DEFAULT, null, trackingActionListener, null); assertSame(validationException, trackingActionListener.exception.get()); } } @@ -307,13 +307,13 @@ public void testPerformRequestOnSuccess() throws IOException { Response mockResponse = new Response(REQUEST_LINE, new HttpHost("localhost", 9200), httpResponse); when(restClient.performRequest(any(Request.class))).thenReturn(mockResponse); { - Integer result = restHighLevelClient.performRequest(mainRequest, requestConverter, + Integer result = restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> response.getStatusLine().getStatusCode(), Collections.emptySet()); assertEquals(restStatus.getStatus(), result.intValue()); } { IOException ioe = expectThrows(IOException.class, () -> restHighLevelClient.performRequest(mainRequest, - requestConverter, response -> {throw new IllegalStateException();}, Collections.emptySet())); + requestConverter, RequestOptions.DEFAULT, response -> {throw new IllegalStateException();}, Collections.emptySet())); assertEquals("Unable to parse response body for Response{requestLine=GET / http/1.1, host=http://localhost:9200, " + "response=http/1.1 " + restStatus.getStatus() + " " + restStatus.name() + "}", ioe.getMessage()); } @@ -328,7 +328,7 @@ public void testPerformRequestOnResponseExceptionWithoutEntity() throws IOExcept ResponseException responseException = new ResponseException(mockResponse); when(restClient.performRequest(any(Request.class))).thenThrow(responseException); ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); assertEquals(responseException.getMessage(), elasticsearchException.getMessage()); assertEquals(restStatus, elasticsearchException.status()); @@ -346,7 +346,7 @@ public void testPerformRequestOnResponseExceptionWithEntity() throws IOException ResponseException responseException = new ResponseException(mockResponse); when(restClient.performRequest(any(Request.class))).thenThrow(responseException); ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); assertEquals("Elasticsearch exception [type=exception, reason=test error message]", elasticsearchException.getMessage()); assertEquals(restStatus, elasticsearchException.status()); @@ -363,7 +363,7 @@ public void testPerformRequestOnResponseExceptionWithBrokenEntity() throws IOExc ResponseException responseException = new ResponseException(mockResponse); when(restClient.performRequest(any(Request.class))).thenThrow(responseException); ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); assertEquals("Unable to parse response body", elasticsearchException.getMessage()); assertEquals(restStatus, elasticsearchException.status()); @@ -381,7 +381,7 @@ public void testPerformRequestOnResponseExceptionWithBrokenEntity2() throws IOEx ResponseException responseException = new ResponseException(mockResponse); when(restClient.performRequest(any(Request.class))).thenThrow(responseException); ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> response.getStatusLine().getStatusCode(), Collections.emptySet())); assertEquals("Unable to parse response body", elasticsearchException.getMessage()); assertEquals(restStatus, elasticsearchException.status()); @@ -397,7 +397,7 @@ public void testPerformRequestOnResponseExceptionWithIgnores() throws IOExceptio ResponseException responseException = new ResponseException(mockResponse); when(restClient.performRequest(any(Request.class))).thenThrow(responseException); //although we got an exception, we turn it into a successful response because the status code was provided among ignores - assertEquals(Integer.valueOf(404), restHighLevelClient.performRequest(mainRequest, requestConverter, + assertEquals(Integer.valueOf(404), restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> response.getStatusLine().getStatusCode(), Collections.singleton(404))); } @@ -409,7 +409,7 @@ public void testPerformRequestOnResponseExceptionWithIgnoresErrorNoBody() throws ResponseException responseException = new ResponseException(mockResponse); when(restClient.performRequest(any(Request.class))).thenThrow(responseException); ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> {throw new IllegalStateException();}, Collections.singleton(404))); assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); assertSame(responseException, elasticsearchException.getCause()); @@ -426,7 +426,7 @@ public void testPerformRequestOnResponseExceptionWithIgnoresErrorValidBody() thr ResponseException responseException = new ResponseException(mockResponse); when(restClient.performRequest(any(Request.class))).thenThrow(responseException); ElasticsearchException elasticsearchException = expectThrows(ElasticsearchException.class, - () -> restHighLevelClient.performRequest(mainRequest, requestConverter, + () -> restHighLevelClient.performRequest(mainRequest, requestConverter, RequestOptions.DEFAULT, response -> {throw new IllegalStateException();}, Collections.singleton(404))); assertEquals(RestStatus.NOT_FOUND, elasticsearchException.status()); assertSame(responseException, elasticsearchException.getSuppressed()[0]); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 80d09acf2817d..a87aec7c2cf87 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -164,8 +164,7 @@ public void testSearchNoQuery() throws IOException { public void testSearchMatchQuery() throws IOException { SearchRequest searchRequest = new SearchRequest("index"); searchRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getAggregations()); assertNull(searchResponse.getSuggest()); @@ -191,8 +190,7 @@ public void testSearchWithTermsAgg() throws IOException { searchSourceBuilder.aggregation(new TermsAggregationBuilder("agg1", ValueType.STRING).field("type.keyword")); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -218,8 +216,7 @@ public void testSearchWithRangeAgg() throws IOException { searchRequest.source(searchSourceBuilder); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, - () -> execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync)); + () -> execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync)); assertEquals(RestStatus.BAD_REQUEST, exception.status()); } @@ -229,8 +226,7 @@ public void testSearchWithRangeAgg() throws IOException { .addRange("first", 0, 30).addRange("second", 31, 200)); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -261,8 +257,7 @@ public void testSearchWithTermsAndRangeAgg() throws IOException { searchSourceBuilder.aggregation(agg); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -313,8 +308,7 @@ public void testSearchWithMatrixStats() throws IOException { searchSourceBuilder.aggregation(new MatrixStatsAggregationBuilder("agg1").fields(Arrays.asList("num", "num2"))); searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -403,8 +397,7 @@ public void testSearchWithParentJoin() throws IOException { SearchRequest searchRequest = new SearchRequest(indexName); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getSuggest()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -444,8 +437,7 @@ public void testSearchWithSuggest() throws IOException { searchSourceBuilder.size(0); searchRequest.source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); assertSearchHeader(searchResponse); assertNull(searchResponse.getAggregations()); assertEquals(Collections.emptyMap(), searchResponse.getProfileResults()); @@ -477,8 +469,7 @@ public void testSearchWithWeirdScriptFields() throws Exception { { SearchRequest searchRequest = new SearchRequest("test").source(SearchSourceBuilder.searchSource() .scriptField("result", new Script("null"))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); SearchHit searchHit = searchResponse.getHits().getAt(0); List values = searchHit.getFields().get("result").getValues(); assertNotNull(values); @@ -488,8 +479,7 @@ public void testSearchWithWeirdScriptFields() throws Exception { { SearchRequest searchRequest = new SearchRequest("test").source(SearchSourceBuilder.searchSource() .scriptField("result", new Script("new HashMap()"))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); SearchHit searchHit = searchResponse.getHits().getAt(0); List values = searchHit.getFields().get("result").getValues(); assertNotNull(values); @@ -501,8 +491,7 @@ public void testSearchWithWeirdScriptFields() throws Exception { { SearchRequest searchRequest = new SearchRequest("test").source(SearchSourceBuilder.searchSource() .scriptField("result", new Script("new String[]{}"))); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); SearchHit searchHit = searchResponse.getHits().getAt(0); List values = searchHit.getFields().get("result").getValues(); assertNotNull(values); @@ -524,8 +513,7 @@ public void testSearchScroll() throws Exception { SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(35).sort("field", SortOrder.ASC); SearchRequest searchRequest = new SearchRequest("test").scroll(TimeValue.timeValueMinutes(2)).source(searchSourceBuilder); - SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync, - highLevelClient()::search, highLevelClient()::searchAsync); + SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync); try { long counter = 0; @@ -537,7 +525,6 @@ public void testSearchScroll() throws Exception { } searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)), - highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync, highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync); assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L)); @@ -547,7 +534,6 @@ public void testSearchScroll() throws Exception { } searchResponse = execute(new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)), - highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync, highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync); assertThat(searchResponse.getHits().getTotalHits(), equalTo(100L)); @@ -559,14 +545,12 @@ public void testSearchScroll() throws Exception { ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); clearScrollRequest.addScrollId(searchResponse.getScrollId()); ClearScrollResponse clearScrollResponse = execute(clearScrollRequest, - highLevelClient()::clearScroll, highLevelClient()::clearScrollAsync, highLevelClient()::clearScroll, highLevelClient()::clearScrollAsync); assertThat(clearScrollResponse.getNumFreed(), greaterThan(0)); assertTrue(clearScrollResponse.isSucceeded()); SearchScrollRequest scrollRequest = new SearchScrollRequest(searchResponse.getScrollId()).scroll(TimeValue.timeValueMinutes(2)); ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, () -> execute(scrollRequest, - highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync, highLevelClient()::searchScroll, highLevelClient()::searchScrollAsync)); assertEquals(RestStatus.NOT_FOUND, exception.status()); assertThat(exception.getRootCause(), instanceOf(ElasticsearchException.class)); @@ -588,8 +572,7 @@ public void testMultiSearch() throws Exception { multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, - highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3)); @@ -631,8 +614,7 @@ public void testMultiSearch_withAgg() throws Exception { multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, - highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3)); @@ -680,8 +662,7 @@ public void testMultiSearch_withQuery() throws Exception { multiSearchRequest.add(searchRequest3); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, - highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(3)); @@ -744,8 +725,7 @@ public void testMultiSearch_failure() throws Exception { multiSearchRequest.add(searchRequest2); MultiSearchResponse multiSearchResponse = - execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync, - highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); + execute(multiSearchRequest, highLevelClient()::multiSearch, highLevelClient()::multiSearchAsync); assertThat(multiSearchResponse.getTook().millis(), Matchers.greaterThanOrEqualTo(0L)); assertThat(multiSearchResponse.getResponses().length, Matchers.equalTo(2)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index ef92e28a07280..4193685f14bc2 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -746,7 +746,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::bulk-execute-async - client.bulkAsync(request, listener); // <1> + client.bulkAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::bulk-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); @@ -995,8 +995,9 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) } }; - BulkProcessor bulkProcessor = - BulkProcessor.builder(client::bulkAsync, listener).build(); // <5> + BulkProcessor bulkProcessor = BulkProcessor.builder( + (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), + listener).build(); // <5> // end::bulk-processor-init assertNotNull(bulkProcessor); @@ -1054,7 +1055,8 @@ public void afterBulk(long executionId, BulkRequest request, Throwable failure) // end::bulk-processor-listener // tag::bulk-processor-options - BulkProcessor.Builder builder = BulkProcessor.builder(client::bulkAsync, listener); + BulkProcessor.Builder builder = BulkProcessor.builder( + (request, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener), listener); builder.setBulkActions(500); // <1> builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB)); // <2> builder.setConcurrentRequests(0); // <3> @@ -1175,7 +1177,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::multi-get-execute-async - client.multiGetAsync(request, listener); // <1> + client.multiGetAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::multi-get-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java index fffba8e5f3f58..7bd6b16cecc99 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/IndicesClientDocumentationIT.java @@ -576,7 +576,7 @@ public void testGetMapping() throws IOException { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); PutMappingRequest request = new PutMappingRequest("twitter"); request.type("tweet"); @@ -589,7 +589,7 @@ public void testGetMapping() throws IOException { " }\n" + "}", // <1> XContentType.JSON); - PutMappingResponse putMappingResponse = client.indices().putMapping(request); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); assertTrue(putMappingResponse.isAcknowledged()); } @@ -633,7 +633,7 @@ public void testGetMappingAsync() throws Exception { final RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter")); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("twitter"), RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); PutMappingRequest request = new PutMappingRequest("twitter"); request.type("tweet"); @@ -646,7 +646,7 @@ public void testGetMappingAsync() throws Exception { " }\n" + "}", // <1> XContentType.JSON); - PutMappingResponse putMappingResponse = client.indices().putMapping(request); + PutMappingResponse putMappingResponse = client.indices().putMapping(request, RequestOptions.DEFAULT); assertTrue(putMappingResponse.isAcknowledged()); } @@ -1733,7 +1733,8 @@ public void testGetAlias() throws Exception { RestHighLevelClient client = highLevelClient(); { - CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index").alias(new Alias("alias"))); + CreateIndexResponse createIndexResponse = client.indices().create(new CreateIndexRequest("index").alias(new Alias("alias")), + RequestOptions.DEFAULT); assertTrue(createIndexResponse.isAcknowledged()); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index cf6409bab6444..adc0fede1aa78 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -628,7 +628,7 @@ public void onFailure(Exception e) { scrollListener = new LatchedActionListener<>(scrollListener, latch); // tag::search-scroll-execute-async - client.searchScrollAsync(scrollRequest, scrollListener); // <1> + client.searchScrollAsync(scrollRequest, RequestOptions.DEFAULT, scrollListener); // <1> // end::search-scroll-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/distribution/src/bin/elasticsearch-cli.bat b/distribution/src/bin/elasticsearch-cli.bat index b668a7c06c272..e17ade3b74af1 100644 --- a/distribution/src/bin/elasticsearch-cli.bat +++ b/distribution/src/bin/elasticsearch-cli.bat @@ -2,7 +2,7 @@ call "%~dp0elasticsearch-env.bat" || exit /b 1 if defined ES_ADDITIONAL_SOURCES ( for %%a in ("%ES_ADDITIONAL_SOURCES:;=","%") do ( - call %~dp0%%a + call "%~dp0%%a" ) ) diff --git a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java index 6a3f57c98d205..3c54afb92c7b7 100644 --- a/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java +++ b/distribution/tools/plugin-cli/src/main/java/org/elasticsearch/plugins/InstallPluginCommand.java @@ -23,6 +23,7 @@ import joptsimple.OptionSpec; import org.apache.lucene.search.spell.LevensteinDistance; import org.apache.lucene.util.CollectionUtil; +import org.bouncycastle.bcpg.ArmoredInputStream; import org.bouncycastle.jce.provider.BouncyCastleProvider; import org.bouncycastle.openpgp.PGPException; import org.bouncycastle.openpgp.PGPPublicKey; @@ -47,7 +48,6 @@ import org.elasticsearch.env.Environment; import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -74,7 +74,6 @@ import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -543,8 +542,8 @@ void verifySignature(final Path zip, final String urlString) throws IOException, InputStream fin = pluginZipInputStream(zip); // sin is a URL stream to the signature corresponding to the downloaded plugin zip InputStream sin = urlOpenStream(ascUrl); - // pin is a input stream to the public key in ASCII-Armor format (RFC4880); the Armor data is in RFC2045 format - InputStream pin = getPublicKey()) { + // ain is a input stream to the public key in ASCII-Armor format (RFC4880) + InputStream ain = new ArmoredInputStream(getPublicKey())) { final JcaPGPObjectFactory factory = new JcaPGPObjectFactory(PGPUtil.getDecoderStream(sin)); final PGPSignature signature = ((PGPSignatureList) factory.nextObject()).get(0); @@ -555,18 +554,6 @@ void verifySignature(final Path zip, final String urlString) throws IOException, } // compute the signature of the downloaded plugin zip - final List lines = - new BufferedReader(new InputStreamReader(pin, StandardCharsets.UTF_8)).lines().collect(Collectors.toList()); - // skip armor headers and possible blank line - int index = 1; - for (; index < lines.size(); index++) { - if (lines.get(index).matches(".*: .*") == false && lines.get(index).matches("\\s*") == false) { - break; - } - } - final byte[] armoredData = - lines.subList(index, lines.size() - 1).stream().collect(Collectors.joining("\n")).getBytes(StandardCharsets.UTF_8); - final InputStream ain = Base64.getMimeDecoder().wrap(new ByteArrayInputStream(armoredData)); final PGPPublicKeyRingCollection collection = new PGPPublicKeyRingCollection(ain, new JcaKeyFingerprintCalculator()); final PGPPublicKey key = collection.getPublicKey(signature.getKeyID()); signature.init(new JcaPGPContentVerifierBuilderProvider().setProvider(new BouncyCastleProvider()), key); diff --git a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java index e9d0974c1438c..1db551934c768 100644 --- a/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java +++ b/distribution/tools/plugin-cli/src/test/java/org/elasticsearch/plugins/InstallPluginCommandTests.java @@ -23,7 +23,6 @@ import com.google.common.jimfs.Configuration; import com.google.common.jimfs.Jimfs; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.LuceneTestCase.AwaitsFix; import org.bouncycastle.bcpg.ArmoredOutputStream; import org.bouncycastle.bcpg.BCPGOutputStream; import org.bouncycastle.bcpg.HashAlgorithmTags; @@ -116,7 +115,6 @@ import static org.hamcrest.Matchers.not; @LuceneTestCase.SuppressFileSystems("*") -@AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/30900") public class InstallPluginCommandTests extends ESTestCase { private InstallPluginCommand skipJarHellCommand; diff --git a/docs/java-rest/high-level/cluster/health.asciidoc b/docs/java-rest/high-level/cluster/health.asciidoc index 6c0f926f15f42..192880849e26d 100644 --- a/docs/java-rest/high-level/cluster/health.asciidoc +++ b/docs/java-rest/high-level/cluster/health.asciidoc @@ -67,6 +67,7 @@ include-tagged::{doc-tests}/ClusterClientDocumentationIT.java[health-request-wai include-tagged::{doc-tests}/ClusterClientDocumentationIT.java[health-request-level] -------------------------------------------------- <1> The level of detail of the returned health information. Accepts a `ClusterHealthRequest.Level` value. +Default value is `cluster`. ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- diff --git a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc index daa86969e4556..1a4d6d4774c49 100644 --- a/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc +++ b/docs/reference/aggregations/metrics/scripted-metric-aggregation.asciidoc @@ -1,8 +1,6 @@ [[search-aggregations-metrics-scripted-metric-aggregation]] === Scripted Metric Aggregation -experimental[] - A metric aggregation that executes using scripts to provide a metric output. Example: diff --git a/docs/reference/migration/migrate_7_0.asciidoc b/docs/reference/migration/migrate_7_0.asciidoc index 6f008a275c737..c80b0ae64a371 100644 --- a/docs/reference/migration/migrate_7_0.asciidoc +++ b/docs/reference/migration/migrate_7_0.asciidoc @@ -36,6 +36,7 @@ Elasticsearch 6.x in order to be readable by Elasticsearch 7.x. * <> * <> * <> +* <> include::migrate_7_0/aggregations.asciidoc[] include::migrate_7_0/analysis.asciidoc[] @@ -49,4 +50,5 @@ include::migrate_7_0/api.asciidoc[] include::migrate_7_0/java.asciidoc[] include::migrate_7_0/settings.asciidoc[] include::migrate_7_0/scripting.asciidoc[] -include::migrate_7_0/snapshotstats.asciidoc[] \ No newline at end of file +include::migrate_7_0/snapshotstats.asciidoc[] +include::migrate_7_0/restclient.asciidoc[] \ No newline at end of file diff --git a/docs/reference/migration/migrate_7_0/restclient.asciidoc b/docs/reference/migration/migrate_7_0/restclient.asciidoc new file mode 100644 index 0000000000000..470996cfeffec --- /dev/null +++ b/docs/reference/migration/migrate_7_0/restclient.asciidoc @@ -0,0 +1,20 @@ +[[breaking_70_restclient_changes]] +=== High-level REST client changes + +==== API methods accepting `Header` argument have been removed + +All API methods accepting headers as a `Header` varargs argument, deprecated +since 6.4, have been removed in favour of the newly introduced methods that +accept instead a `RequestOptions` argument. In case you are not specifying any +header, e.g. `client.index(indexRequest)` becomes +`client.index(indexRequest, RequestOptions.DEFAULT)`. +In case you are specifying headers +e.g. `client.index(indexRequest, new Header("name" "value"))` becomes +`client.index(indexRequest, RequestOptions.DEFAULT.toBuilder().addHeader("name", "value").build());` + +==== Cluster Health API default to `cluster` level + +The Cluster Health API used to default to `shards` level to ease migration +from transport client that doesn't support the `level` parameter and always +returns information including indices and shards details. The level default +value has been aligned with the Elasticsearch default level: `cluster`. \ No newline at end of file diff --git a/docs/reference/settings/audit-settings.asciidoc b/docs/reference/settings/audit-settings.asciidoc index 5995c65a01c9f..524198df58c47 100644 --- a/docs/reference/settings/audit-settings.asciidoc +++ b/docs/reference/settings/audit-settings.asciidoc @@ -112,6 +112,15 @@ xpack.security.audit.index.settings: number_of_replicas: 1 ---------------------------- -- ++ +-- +NOTE: These settings apply to the local audit indices, as well as to the +<>, but only if the remote cluster +does *not* have {security} installed, or the {es} versions are different. +If the remote cluster has {security} installed, and the versions coincide, the +settings for the audit indices there will take precedence, +even if they are unspecified (i.e. left to defaults). +-- [[remote-audit-settings]] ==== Remote Audit Log Indexing Configuration Settings diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/ChannelContext.java b/libs/nio/src/main/java/org/elasticsearch/nio/ChannelContext.java index e3702c2880a18..a7cb21d95f537 100644 --- a/libs/nio/src/main/java/org/elasticsearch/nio/ChannelContext.java +++ b/libs/nio/src/main/java/org/elasticsearch/nio/ChannelContext.java @@ -66,7 +66,7 @@ protected void setSelectionKey(SelectionKey selectionKey) { * @throws IOException during channel / context close */ public void closeFromSelector() throws IOException { - if (closeContext.isDone() == false) { + if (isOpen()) { try { rawChannel.close(); closeContext.complete(null); diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/EventHandler.java b/libs/nio/src/main/java/org/elasticsearch/nio/EventHandler.java index 3c52423c7aff3..87a2489fdbc27 100644 --- a/libs/nio/src/main/java/org/elasticsearch/nio/EventHandler.java +++ b/libs/nio/src/main/java/org/elasticsearch/nio/EventHandler.java @@ -159,8 +159,7 @@ protected void listenerException(Exception exception) { } /** - * This method is called after ready events (READ, ACCEPT, WRITE, CONNECT) have been handled for a - * channel. + * This method is called after events (READ, WRITE, CONNECT) have been handled for a channel. * * @param context that was handled */ diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java b/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java index ab6709bcc5bd4..9f82cc2c50d44 100644 --- a/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java +++ b/libs/nio/src/main/java/org/elasticsearch/nio/NioSelector.java @@ -43,9 +43,6 @@ * {@link #runLoop()}, the selector will run until {@link #close()} is called. This instance handles closing * of channels. Users should call {@link #queueChannelClose(NioChannel)} to schedule a channel for close by * this selector. - *

- * Children of this class should implement the specific {@link #processKey(SelectionKey)}, - * {@link #preSelect()}, and {@link #cleanup()} functionality. */ public class NioSelector implements Closeable { @@ -65,7 +62,7 @@ public NioSelector(EventHandler eventHandler) throws IOException { this(eventHandler, Selector.open()); } - public NioSelector(EventHandler eventHandler, Selector selector) throws IOException { + public NioSelector(EventHandler eventHandler, Selector selector) { this.selector = selector; this.eventHandler = eventHandler; } @@ -165,7 +162,7 @@ void singleLoop() { } void cleanupAndCloseChannels() { - cleanup(); + cleanupPendingWrites(); channelsToClose.addAll(channelsToRegister); channelsToRegister.clear(); channelsToClose.addAll(selector.keys().stream().map(sk -> (ChannelContext) sk.attachment()).collect(Collectors.toList())); @@ -234,16 +231,6 @@ void preSelect() { handleQueuedWrites(); } - /** - * Called once as the selector is being closed. - */ - void cleanup() { - WriteOperation op; - while ((op = queuedWrites.poll()) != null) { - executeFailedListener(op.getListener(), new ClosedSelectorException()); - } - } - /** * Queues a write operation to be handled by the event loop. This can be called by any thread and is the * api available for non-selector threads to schedule writes. @@ -284,20 +271,31 @@ public void scheduleForRegistration(NioChannel channel) { } /** - * Queues a write operation directly in a channel's buffer. Channel buffers are only safe to be accessed - * by the selector thread. As a result, this method should only be called by the selector thread. + * Queues a write operation directly in a channel's buffer. If this channel does not have pending writes + * already, the channel will be flushed. Channel buffers are only safe to be accessed by the selector + * thread. As a result, this method should only be called by the selector thread. If this channel does + * not have pending writes already, the channel will be flushed. * * @param writeOperation to be queued in a channel's buffer */ - public void queueWriteInChannelBuffer(WriteOperation writeOperation) { + public void writeToChannel(WriteOperation writeOperation) { assertOnSelectorThread(); SocketChannelContext context = writeOperation.getChannel(); + // If the channel does not currently have anything that is ready to flush, we should flush after + // the write operation is queued. + boolean shouldFlushAfterQueuing = context.readyForFlush() == false; try { SelectionKeyUtils.setWriteInterested(context.getSelectionKey()); context.queueWriteOperation(writeOperation); } catch (Exception e) { + shouldFlushAfterQueuing = false; executeFailedListener(writeOperation.getListener(), e); } + + if (shouldFlushAfterQueuing) { + handleWrite(context); + eventHandler.postHandling(context); + } } /** @@ -332,6 +330,13 @@ public void executeFailedListener(BiConsumer listener, Excepti } } + private void cleanupPendingWrites() { + WriteOperation op; + while ((op = queuedWrites.poll()) != null) { + executeFailedListener(op.getListener(), new ClosedSelectorException()); + } + } + private void wakeup() { // TODO: Do we need the wakeup optimizations that some other libraries use? selector.wakeup(); @@ -394,7 +399,7 @@ private void handleQueuedWrites() { WriteOperation writeOperation; while ((writeOperation = queuedWrites.poll()) != null) { if (writeOperation.getChannel().isOpen()) { - queueWriteInChannelBuffer(writeOperation); + writeToChannel(writeOperation); } else { executeFailedListener(writeOperation.getListener(), new ClosedChannelException()); } diff --git a/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java b/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java index 53be0e7f89fe0..53fb0da432f48 100644 --- a/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java +++ b/libs/nio/src/main/java/org/elasticsearch/nio/SocketChannelContext.java @@ -135,7 +135,7 @@ public void sendMessage(Object message, BiConsumer listener) { return; } - selector.queueWriteInChannelBuffer(writeOperation); + selector.writeToChannel(writeOperation); } public void queueWriteOperation(WriteOperation writeOperation) { @@ -164,7 +164,7 @@ protected FlushOperation getPendingFlush() { @Override public void closeFromSelector() throws IOException { getSelector().assertOnSelectorThread(); - if (channel.isOpen()) { + if (isOpen()) { ArrayList closingExceptions = new ArrayList<>(3); try { super.closeFromSelector(); diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java index dd3fea8bf50e8..bd5f1c1eb346f 100644 --- a/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java +++ b/libs/nio/src/test/java/org/elasticsearch/nio/NioSelectorTests.java @@ -262,11 +262,28 @@ public void testQueueWriteSuccessful() throws Exception { public void testQueueDirectlyInChannelBufferSuccessful() throws Exception { WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener); - assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) == 0); + assertEquals(0, (selectionKey.interestOps() & SelectionKey.OP_WRITE)); - selector.queueWriteInChannelBuffer(writeOperation); + when(channelContext.readyForFlush()).thenReturn(true); + selector.writeToChannel(writeOperation); verify(channelContext).queueWriteOperation(writeOperation); + verify(eventHandler, times(0)).handleWrite(channelContext); + verify(eventHandler, times(0)).postHandling(channelContext); + assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) != 0); + } + + public void testShouldFlushIfNoPendingFlushes() throws Exception { + WriteOperation writeOperation = new FlushReadyWrite(channelContext, buffers, listener); + + assertEquals(0, (selectionKey.interestOps() & SelectionKey.OP_WRITE)); + + when(channelContext.readyForFlush()).thenReturn(false); + selector.writeToChannel(writeOperation); + + verify(channelContext).queueWriteOperation(writeOperation); + verify(eventHandler).handleWrite(channelContext); + verify(eventHandler).postHandling(channelContext); assertTrue((selectionKey.interestOps() & SelectionKey.OP_WRITE) != 0); } @@ -277,10 +294,13 @@ public void testQueueDirectlyInChannelBufferSelectionKeyThrowsException() throws CancelledKeyException cancelledKeyException = new CancelledKeyException(); when(channelContext.getSelectionKey()).thenReturn(selectionKey); + when(channelContext.readyForFlush()).thenReturn(false); when(selectionKey.interestOps(anyInt())).thenThrow(cancelledKeyException); - selector.queueWriteInChannelBuffer(writeOperation); + selector.writeToChannel(writeOperation); verify(channelContext, times(0)).queueWriteOperation(writeOperation); + verify(eventHandler, times(0)).handleWrite(channelContext); + verify(eventHandler, times(0)).postHandling(channelContext); verify(listener).accept(null, cancelledKeyException); } diff --git a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java index fdb4a77b922e2..dee50724f34c9 100644 --- a/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java +++ b/libs/nio/src/test/java/org/elasticsearch/nio/SocketChannelContextTests.java @@ -170,7 +170,7 @@ public void testSendMessageFromSameThreadIsQueuedInChannel() { when(readWriteHandler.createWriteOperation(context, buffers, listener)).thenReturn(writeOperation); context.sendMessage(buffers, listener); - verify(selector).queueWriteInChannelBuffer(writeOpCaptor.capture()); + verify(selector).writeToChannel(writeOpCaptor.capture()); WriteOperation writeOp = writeOpCaptor.getValue(); assertSame(writeOperation, writeOp); diff --git a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java index 24dce7abcf370..722d75a9293f7 100644 --- a/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java +++ b/modules/analysis-common/src/main/java/org/elasticsearch/analysis/common/CommonAnalysisPlugin.java @@ -247,108 +247,31 @@ public Map> getTokenizers() { @Override public List getPreBuiltAnalyzerProviderFactories() { List analyzers = new ArrayList<>(); - analyzers.add(new PreBuiltAnalyzerProviderFactory("standard_html_strip", CachingStrategy.LUCENE, version -> { - Analyzer a = new StandardHtmlStripAnalyzer(CharArraySet.EMPTY_SET); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("pattern", CachingStrategy.ELASTICSEARCH, version -> { - Analyzer a = new PatternAnalyzer(Regex.compile("\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/, null), true, - CharArraySet.EMPTY_SET); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("snowball", CachingStrategy.LUCENE, version -> { - Analyzer a = new SnowballAnalyzer("English", StopAnalyzer.ENGLISH_STOP_WORDS_SET); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("arabic", CachingStrategy.LUCENE, version -> { - Analyzer a = new ArabicAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("armenian", CachingStrategy.LUCENE, version -> { - Analyzer a = new ArmenianAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("basque", CachingStrategy.LUCENE, version -> { - Analyzer a = new BasqueAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("bengali", CachingStrategy.LUCENE, version -> { - Analyzer a = new BengaliAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("brazilian", CachingStrategy.LUCENE, version -> { - Analyzer a = new BrazilianAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("bulgarian", CachingStrategy.LUCENE, version -> { - Analyzer a = new BulgarianAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("catalan", CachingStrategy.LUCENE, version -> { - Analyzer a = new CatalanAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("chinese", CachingStrategy.LUCENE, version -> { - // only for old indices, best effort - Analyzer a = new StandardAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("cjk", CachingStrategy.LUCENE, version -> { - Analyzer a = new CJKAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("czech", CachingStrategy.LUCENE, version -> { - Analyzer a = new CzechAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("danish", CachingStrategy.LUCENE, version -> { - Analyzer a = new DanishAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("dutch", CachingStrategy.LUCENE, version -> { - Analyzer a = new DutchAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("english", CachingStrategy.LUCENE, version -> { - Analyzer a = new EnglishAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("finnish", CachingStrategy.LUCENE, version -> { - Analyzer a = new FinnishAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("french", CachingStrategy.LUCENE, version -> { - Analyzer a = new FrenchAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("galician", CachingStrategy.LUCENE, version -> { - Analyzer a = new GalicianAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); - analyzers.add(new PreBuiltAnalyzerProviderFactory("german", CachingStrategy.LUCENE, version -> { - Analyzer a = new GermanAnalyzer(); - a.setVersion(version.luceneVersion); - return a; - })); + analyzers.add(new PreBuiltAnalyzerProviderFactory("standard_html_strip", CachingStrategy.LUCENE, + () -> new StandardHtmlStripAnalyzer(CharArraySet.EMPTY_SET))); + analyzers.add(new PreBuiltAnalyzerProviderFactory("pattern", CachingStrategy.ELASTICSEARCH, + () -> new PatternAnalyzer(Regex.compile("\\W+" /*PatternAnalyzer.NON_WORD_PATTERN*/, null), true, + CharArraySet.EMPTY_SET))); + analyzers.add(new PreBuiltAnalyzerProviderFactory("snowball", CachingStrategy.LUCENE, + () -> new SnowballAnalyzer("English", StopAnalyzer.ENGLISH_STOP_WORDS_SET))); + analyzers.add(new PreBuiltAnalyzerProviderFactory("arabic", CachingStrategy.LUCENE, ArabicAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("armenian", CachingStrategy.LUCENE, ArmenianAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("basque", CachingStrategy.LUCENE, BasqueAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("bengali", CachingStrategy.LUCENE, BengaliAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("brazilian", CachingStrategy.LUCENE, BrazilianAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("bulgarian", CachingStrategy.LUCENE, BulgarianAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("catalan", CachingStrategy.LUCENE, CatalanAnalyzer::new)); + // chinese analyzer: only for old indices, best effort + analyzers.add(new PreBuiltAnalyzerProviderFactory("chinese", CachingStrategy.LUCENE, StandardAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("cjk", CachingStrategy.LUCENE, CJKAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("czech", CachingStrategy.LUCENE, CzechAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("danish", CachingStrategy.LUCENE, DanishAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("dutch", CachingStrategy.LUCENE, DutchAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("english", CachingStrategy.LUCENE, EnglishAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("finnish", CachingStrategy.LUCENE, FinnishAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("french", CachingStrategy.LUCENE, FrenchAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("galician", CachingStrategy.LUCENE, GalicianAnalyzer::new)); + analyzers.add(new PreBuiltAnalyzerProviderFactory("german", CachingStrategy.LUCENE, GermanAnalyzer::new)); return analyzers; } diff --git a/modules/reindex/build.gradle b/modules/reindex/build.gradle index 765d55dd095c7..8870e21858d18 100644 --- a/modules/reindex/build.gradle +++ b/modules/reindex/build.gradle @@ -121,6 +121,11 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { baseDir, unzip.temporaryDir, version == '090' + waitCondition = { fixture, ant -> + // the fixture writes the ports file when Elasticsearch's HTTP service + // is ready, so we can just wait for the file to exist + return fixture.portsFile.exists() + } } integTest.dependsOn fixture integTestRunner { diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java index 940918de10dc6..72ba651dff9ae 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ReindexFromRemoteWithAuthTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.reindex; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.ElasticsearchStatusException; @@ -42,7 +41,6 @@ import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.env.Environment; import org.elasticsearch.env.NodeEnvironment; -import org.elasticsearch.index.reindex.test.ObjectCleanerThreadThreadFilter; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; @@ -66,7 +64,6 @@ import static org.elasticsearch.index.reindex.ReindexTestCase.matcher; import static org.hamcrest.Matchers.containsString; -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) public class ReindexFromRemoteWithAuthTests extends ESSingleNodeTestCase { private TransportAddress address; diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java index df00083587018..1107a36086927 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/RetryTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.index.reindex; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; @@ -33,7 +32,6 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.common.util.concurrent.EsRejectedExecutionException; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.reindex.test.ObjectCleanerThreadThreadFilter; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; @@ -57,7 +55,6 @@ * Integration test for retry behavior. Useful because retrying relies on the way that the * rest of Elasticsearch throws exceptions and unit tests won't verify that. */ -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) public class RetryTests extends ESIntegTestCase { private static final int DOC_COUNT = 20; diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java index c9a36ec859021..353a0d895c2c7 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/URLFixture.java @@ -39,6 +39,7 @@ import java.util.Map; import java.util.Objects; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonMap; @@ -67,7 +68,6 @@ public static void main(String[] args) throws Exception { writeFile(workingDirectory, "ports", addressAndPort); // Exposes the repository over HTTP - final String url = "http://" + addressAndPort; httpServer.createContext("/", new ResponseHandler(dir(args[1]))); httpServer.start(); @@ -110,7 +110,13 @@ static class ResponseHandler implements HttpHandler { @Override public void handle(HttpExchange exchange) throws IOException { Response response; - if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) { + + final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent"); + if (userAgent != null && userAgent.startsWith("Apache Ant")) { + // This is a request made by the AntFixture, just reply "OK" + response = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8)); + + } else if ("GET".equalsIgnoreCase(exchange.getRequestMethod())) { String path = exchange.getRequestURI().toString(); if (path.length() > 0 && path.charAt(0) == '/') { path = path.substring(1); @@ -125,13 +131,13 @@ public void handle(HttpExchange exchange) throws IOException { Map headers = singletonMap("Content-Length", String.valueOf(content.length)); response = new Response(RestStatus.OK, headers, "application/octet-stream", content); } else { - response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain", new byte[0]); + response = new Response(RestStatus.NOT_FOUND, emptyMap(), "text/plain; charset=utf-8", new byte[0]); } } else { - response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain", new byte[0]); + response = new Response(RestStatus.FORBIDDEN, emptyMap(), "text/plain; charset=utf-8", new byte[0]); } } else { - response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain", + response = new Response(RestStatus.INTERNAL_SERVER_ERROR, emptyMap(), "text/plain; charset=utf-8", "Unsupported HTTP method".getBytes(StandardCharsets.UTF_8)); } exchange.sendResponseHeaders(response.status.getStatus(), response.body.length); diff --git a/modules/transport-netty4/build.gradle b/modules/transport-netty4/build.gradle index ed905a530c48e..5d4bcd7c10a84 100644 --- a/modules/transport-netty4/build.gradle +++ b/modules/transport-netty4/build.gradle @@ -34,13 +34,13 @@ compileTestJava.options.compilerArgs << "-Xlint:-cast,-deprecation,-rawtypes,-tr dependencies { // network stack - compile "io.netty:netty-buffer:4.1.25.Final" - compile "io.netty:netty-codec:4.1.25.Final" - compile "io.netty:netty-codec-http:4.1.25.Final" - compile "io.netty:netty-common:4.1.25.Final" - compile "io.netty:netty-handler:4.1.25.Final" - compile "io.netty:netty-resolver:4.1.25.Final" - compile "io.netty:netty-transport:4.1.25.Final" + compile "io.netty:netty-buffer:4.1.16.Final" + compile "io.netty:netty-codec:4.1.16.Final" + compile "io.netty:netty-codec-http:4.1.16.Final" + compile "io.netty:netty-common:4.1.16.Final" + compile "io.netty:netty-handler:4.1.16.Final" + compile "io.netty:netty-resolver:4.1.16.Final" + compile "io.netty:netty-transport:4.1.16.Final" } dependencyLicenses { @@ -161,6 +161,6 @@ thirdPartyAudit.excludes = [ 'org.conscrypt.AllocatedBuffer', 'org.conscrypt.BufferAllocator', - 'org.conscrypt.Conscrypt', + 'org.conscrypt.Conscrypt$Engines', 'org.conscrypt.HandshakeListener' ] diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..c546222971985 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-buffer-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +63b5fa95c74785e16f2c30ce268bc222e35c8cb5 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.25.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.25.Final.jar.sha1 deleted file mode 100644 index 3ca0cbb45ec31..0000000000000 --- a/modules/transport-netty4/licenses/netty-buffer-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f366d0cc87b158ca064d27507127e3cc4eb2f089 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..1e6c241ea0b17 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +d84a1f21768b7309c2954521cf5a1f46c2309eb1 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.25.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.25.Final.jar.sha1 deleted file mode 100644 index 5e2bc85c548dd..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3e465c75bead40d06b5b9c0612b37cf77c548887 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..71c33af1c5fc2 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +d64312378b438dfdad84267c599a053327c6f02a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.25.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.25.Final.jar.sha1 deleted file mode 100644 index 58cb7fd987949..0000000000000 --- a/modules/transport-netty4/licenses/netty-codec-http-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -70888d3f2a829541378f68503ddd52c3193df35a \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..3edf5fcea59b3 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-common-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +177a6b30cca92f6f5f9873c9befd681377a4c328 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-common-4.1.25.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.25.Final.jar.sha1 deleted file mode 100644 index 62f85f8965513..0000000000000 --- a/modules/transport-netty4/licenses/netty-common-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e17d5c05c101fe14536ce3fb34b36c54e04791f6 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..cba27387268d1 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-handler-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.25.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.25.Final.jar.sha1 deleted file mode 100644 index 5391f625a4df0..0000000000000 --- a/modules/transport-netty4/licenses/netty-handler-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ecdfb8fe93a8b75db3ea8746d3437eed845c24bd \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..3571d2ecfdc48 --- /dev/null +++ b/modules/transport-netty4/licenses/netty-resolver-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +f6eb553b53fb3a90a8ac1170697093fed82eae28 \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.25.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.25.Final.jar.sha1 deleted file mode 100644 index 8225fb799e3ff..0000000000000 --- a/modules/transport-netty4/licenses/netty-resolver-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc0965d00746b782b33f419b005cbc130973030d \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..e502d4c77084c --- /dev/null +++ b/modules/transport-netty4/licenses/netty-transport-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b \ No newline at end of file diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.25.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.25.Final.jar.sha1 deleted file mode 100644 index 1049ea4b98bc6..0000000000000 --- a/modules/transport-netty4/licenses/netty-transport-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19a6f1f649894b6705aa9d8cbcced188dff133b0 \ No newline at end of file diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java index f99bccdaf6178..12c2e9a685778 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandler.java @@ -22,7 +22,7 @@ import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.FullHttpRequest; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.http.HttpPipelinedRequest; @@ -53,17 +53,14 @@ public Netty4HttpPipeliningHandler(Logger logger, final int maxEventsHeld) { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - if (msg instanceof LastHttpContent) { - HttpPipelinedRequest pipelinedRequest = aggregator.read(((LastHttpContent) msg)); - ctx.fireChannelRead(pipelinedRequest); - } else { - ctx.fireChannelRead(msg); - } + assert msg instanceof FullHttpRequest : "Invalid message type: " + msg.getClass(); + HttpPipelinedRequest pipelinedRequest = aggregator.read(((FullHttpRequest) msg)); + ctx.fireChannelRead(pipelinedRequest); } @Override public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise promise) { - assert msg instanceof Netty4HttpResponse : "Message must be type: " + Netty4HttpResponse.class; + assert msg instanceof Netty4HttpResponse : "Invalid message type: " + msg.getClass();; Netty4HttpResponse response = (Netty4HttpResponse) msg; boolean success = false; try { diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy index 3775931efb150..32b2dc9bd1540 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -21,8 +21,6 @@ grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - permission java.lang.RuntimePermission "setContextClassLoader"; - // netty makes and accepts socket connections permission java.net.SocketPermission "*", "accept,connect"; }; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4IntegTestCase.java b/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java similarity index 90% rename from modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4IntegTestCase.java rename to modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java index c21b863d196b7..b38cda76c6980 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4IntegTestCase.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/ESNetty4IntegTestCase.java @@ -16,21 +16,19 @@ * specific language governing permissions and limitations * under the License. */ +package org.elasticsearch; -package org.elasticsearch.test; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.transport.netty4.Netty4Transport; import java.util.Arrays; import java.util.Collection; -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) -public abstract class Netty4IntegTestCase extends ESIntegTestCase { +public abstract class ESNetty4IntegTestCase extends ESIntegTestCase { @Override protected boolean ignoreExternalCluster() { diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java index 8baf818975ed8..094f339059876 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4BadRequestTests.java @@ -20,7 +20,6 @@ package org.elasticsearch.http.netty4; import io.netty.handler.codec.http.FullHttpResponse; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; @@ -34,6 +33,7 @@ import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -48,7 +48,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; -public class Netty4BadRequestTests extends Netty4TestCase { +public class Netty4BadRequestTests extends ESTestCase { private NetworkService networkService; private MockBigArrays bigArrays; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java index f7f04cfa540f6..7c5b35a322996 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpChannelTests.java @@ -41,7 +41,6 @@ import io.netty.handler.codec.http.HttpVersion; import io.netty.util.Attribute; import io.netty.util.AttributeKey; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.bytes.ReleasablePagedBytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; @@ -65,6 +64,7 @@ import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.netty4.Netty4Utils; @@ -90,7 +90,7 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; -public class Netty4HttpChannelTests extends Netty4TestCase { +public class Netty4HttpChannelTests extends ESTestCase { private NetworkService networkService; private ThreadPool threadPool; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java index 606b050f4e393..81cf0d1f88908 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpPipeliningHandlerTests.java @@ -36,9 +36,9 @@ import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.codec.http.QueryStringDecoder; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.common.Randomness; import org.elasticsearch.http.HttpPipelinedRequest; +import org.elasticsearch.test.ESTestCase; import org.junit.After; import java.nio.channels.ClosedChannelException; @@ -61,7 +61,7 @@ import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; import static org.hamcrest.core.Is.is; -public class Netty4HttpPipeliningHandlerTests extends Netty4TestCase { +public class Netty4HttpPipeliningHandlerTests extends ESTestCase { private final ExecutorService handlerService = Executors.newFixedThreadPool(randomIntBetween(4, 8)); private final ExecutorService eventLoopService = Executors.newFixedThreadPool(1); @@ -148,38 +148,6 @@ public void testThatPipeliningWorksWhenSlowRequestsInDifferentOrder() throws Int assertTrue(embeddedChannel.isOpen()); } - public void testThatPipeliningWorksWithChunkedRequests() throws InterruptedException { - final int numberOfRequests = randomIntBetween(2, 128); - final EmbeddedChannel embeddedChannel = - new EmbeddedChannel( - new AggregateUrisAndHeadersHandler(), - new Netty4HttpPipeliningHandler(logger, numberOfRequests), - new WorkEmulatorHandler()); - - for (int i = 0; i < numberOfRequests; i++) { - final DefaultHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/" + i); - embeddedChannel.writeInbound(request); - embeddedChannel.writeInbound(LastHttpContent.EMPTY_LAST_CONTENT); - } - - final List latches = new ArrayList<>(); - for (int i = numberOfRequests - 1; i >= 0; i--) { - latches.add(finishRequest(Integer.toString(i))); - } - - for (final CountDownLatch latch : latches) { - latch.await(); - } - - embeddedChannel.flush(); - - for (int i = 0; i < numberOfRequests; i++) { - assertReadHttpMessageHasContent(embeddedChannel, Integer.toString(i)); - } - - assertTrue(embeddedChannel.isOpen()); - } - public void testThatPipeliningClosesConnectionWithTooManyEvents() throws InterruptedException { final int numberOfRequests = randomIntBetween(2, 128); final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new Netty4HttpPipeliningHandler(logger, numberOfRequests), diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java index ec3dbbf3846f0..269773fbb634c 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpRequestSizeLimitIT.java @@ -20,7 +20,7 @@ package org.elasticsearch.http.netty4; import io.netty.handler.codec.http.FullHttpResponse; -import org.elasticsearch.test.Netty4IntegTestCase; +import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -45,7 +45,7 @@ * a single node "cluster". We also force test infrastructure to use the node client instead of the transport client for the same reason. */ @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numClientNodes = 0, numDataNodes = 1, transportClientRatio = 0) -public class Netty4HttpRequestSizeLimitIT extends Netty4IntegTestCase { +public class Netty4HttpRequestSizeLimitIT extends ESNetty4IntegTestCase { private static final ByteSizeValue LIMIT = new ByteSizeValue(2, ByteSizeUnit.KB); diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java index 68b51f8acc13f..f2b28b909187b 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerPipeliningTests.java @@ -32,7 +32,6 @@ import io.netty.handler.codec.http.HttpHeaderNames; import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpVersion; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; @@ -43,6 +42,7 @@ import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.http.NullDispatcher; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -61,7 +61,7 @@ /** * This test just tests, if he pipelining works in general with out any connection the Elasticsearch handler */ -public class Netty4HttpServerPipeliningTests extends Netty4TestCase { +public class Netty4HttpServerPipeliningTests extends ESTestCase { private NetworkService networkService; private ThreadPool threadPool; private MockBigArrays bigArrays; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java index 1df18459a0352..5b22409b92da0 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4HttpServerTransportTests.java @@ -38,7 +38,6 @@ import io.netty.handler.codec.http.HttpResponseStatus; import io.netty.handler.codec.http.HttpUtil; import io.netty.handler.codec.http.HttpVersion; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -60,6 +59,7 @@ import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -94,7 +94,7 @@ /** * Tests for the {@link Netty4HttpServerTransport} class. */ -public class Netty4HttpServerTransportTests extends Netty4TestCase { +public class Netty4HttpServerTransportTests extends ESTestCase { private NetworkService networkService; private ThreadPool threadPool; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java index c90992a69d0b3..ebb91d9663ed5 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/http/netty4/Netty4PipeliningIT.java @@ -20,7 +20,7 @@ package org.elasticsearch.http.netty4; import io.netty.handler.codec.http.FullHttpResponse; -import org.elasticsearch.test.Netty4IntegTestCase; +import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; @@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) -public class Netty4PipeliningIT extends Netty4IntegTestCase { +public class Netty4PipeliningIT extends ESNetty4IntegTestCase { @Override protected boolean addMockHttpTransport() { diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4TestCase.java b/modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4TestCase.java deleted file mode 100644 index df931d61992ef..0000000000000 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/test/Netty4TestCase.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; - -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) -public abstract class Netty4TestCase extends ESTestCase { -} diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/test/ObjectCleanerThreadThreadFilter.java b/modules/transport-netty4/src/test/java/org/elasticsearch/test/ObjectCleanerThreadThreadFilter.java deleted file mode 100644 index e47c536665d13..0000000000000 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/test/ObjectCleanerThreadThreadFilter.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test; - -import com.carrotsearch.randomizedtesting.ThreadFilter; - -/** - * The Netty object cleaner thread is not closeable and it does not terminate in a timely manner. This means that thread leak control in - * tests will fail test suites when the object cleaner thread has not terminated. Since there is not a reliable way to terminate this thread - * we instead filter it out of thread leak control. - */ -public class ObjectCleanerThreadThreadFilter implements ThreadFilter { - - @Override - public boolean reject(final Thread t) { - // TODO: replace with constant from Netty when https://github.com/netty/netty/pull/8014 is integrated - return "ObjectCleanerThread".equals(t.getName()); - } - -} - diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java index 618d3ffe8f96a..4a41aaec952a0 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ByteBufBytesReferenceTests.java @@ -18,18 +18,15 @@ */ package org.elasticsearch.transport.netty4; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import io.netty.buffer.ByteBuf; import io.netty.buffer.Unpooled; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.test.ObjectCleanerThreadThreadFilter; import org.elasticsearch.common.bytes.AbstractBytesReferenceTestCase; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.ReleasableBytesStreamOutput; import java.io.IOException; -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) public class ByteBufBytesReferenceTests extends AbstractBytesReferenceTestCase { @Override diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java index 8ef8b28dc1497..acd71749e2333 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/ESLoggingHandlerIT.java @@ -20,7 +20,7 @@ package org.elasticsearch.transport.netty4; import org.apache.logging.log4j.Level; -import org.elasticsearch.test.Netty4IntegTestCase; +import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.action.admin.cluster.node.hotthreads.NodesHotThreadsRequest; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.test.ESIntegTestCase; @@ -29,7 +29,7 @@ @ESIntegTestCase.ClusterScope(numDataNodes = 2) @TestLogging(value = "org.elasticsearch.transport.netty4.ESLoggingHandler:trace") -public class ESLoggingHandlerIT extends Netty4IntegTestCase { +public class ESLoggingHandlerIT extends ESNetty4IntegTestCase { private MockLogAppender appender; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java index b4b33ba96211c..b967a7ea41069 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4ScheduledPingTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.transport.netty4; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.lease.Releasables; @@ -27,6 +26,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.transport.MockTransportService; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; @@ -47,7 +47,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -public class Netty4ScheduledPingTests extends Netty4TestCase { +public class Netty4ScheduledPingTests extends ESTestCase { public void testScheduledPing() throws Exception { ThreadPool threadPool = new TestThreadPool(getClass().getName()); diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java index 0819548907296..7343da6c3b11a 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4SizeHeaderFrameDecoderTests.java @@ -19,7 +19,6 @@ package org.elasticsearch.transport.netty4; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkService; import org.elasticsearch.common.settings.Settings; @@ -29,6 +28,7 @@ import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.mocksocket.MockSocket; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpTransport; import org.junit.After; @@ -47,7 +47,7 @@ * This test checks, if a HTTP look-alike request (starting with a HTTP method and a space) * actually returns text response instead of just dropping the connection */ -public class Netty4SizeHeaderFrameDecoderTests extends Netty4TestCase { +public class Netty4SizeHeaderFrameDecoderTests extends ESTestCase { private final Settings settings = Settings.builder() .put("node.name", "NettySizeHeaderFrameDecoderTests") diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java index a426c11db8245..b81c8efcb47ee 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportIT.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.transport.netty4; -import org.elasticsearch.test.Netty4IntegTestCase; +import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -54,7 +54,7 @@ import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) -public class Netty4TransportIT extends Netty4IntegTestCase { +public class Netty4TransportIT extends ESNetty4IntegTestCase { // static so we can use it in anonymous classes private static String channelProfileName = null; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java index ddef8abcf0181..52ad32efb5645 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportMultiPortIntegrationIT.java @@ -18,7 +18,7 @@ */ package org.elasticsearch.transport.netty4; -import org.elasticsearch.test.Netty4IntegTestCase; +import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; @@ -48,7 +48,7 @@ import static org.hamcrest.Matchers.lessThanOrEqualTo; @ClusterScope(scope = Scope.SUITE, supportsDedicatedMasters = false, numDataNodes = 1, numClientNodes = 0) -public class Netty4TransportMultiPortIntegrationIT extends Netty4IntegTestCase { +public class Netty4TransportMultiPortIntegrationIT extends ESNetty4IntegTestCase { private static int randomPort = -1; private static String randomPortRange; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java index dee104a73f7d7..922031d3c3dea 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/Netty4TransportPublishAddressIT.java @@ -19,7 +19,7 @@ package org.elasticsearch.transport.netty4; -import org.elasticsearch.test.Netty4IntegTestCase; +import org.elasticsearch.ESNetty4IntegTestCase; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; import org.elasticsearch.common.network.NetworkModule; @@ -41,7 +41,7 @@ * different ports on ipv4 and ipv6. */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0) -public class Netty4TransportPublishAddressIT extends Netty4IntegTestCase { +public class Netty4TransportPublishAddressIT extends ESNetty4IntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java index 05d6d55ac42da..a49df3caaba4e 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/NettyTransportMultiPortTests.java @@ -18,7 +18,6 @@ */ package org.elasticsearch.transport.netty4; -import org.elasticsearch.test.Netty4TestCase; import org.elasticsearch.common.component.Lifecycle; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.network.NetworkService; @@ -28,6 +27,7 @@ import org.elasticsearch.common.util.MockBigArrays; import org.elasticsearch.common.util.MockPageCacheRecycler; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpTransport; @@ -37,7 +37,7 @@ import static org.hamcrest.Matchers.is; -public class NettyTransportMultiPortTests extends Netty4TestCase { +public class NettyTransportMultiPortTests extends ESTestCase { private String host; diff --git a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java index 278b00965c9a5..efa296b6278af 100644 --- a/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java +++ b/modules/transport-netty4/src/test/java/org/elasticsearch/transport/netty4/SimpleNetty4TransportTests.java @@ -19,8 +19,6 @@ package org.elasticsearch.transport.netty4; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import org.elasticsearch.test.ObjectCleanerThreadThreadFilter; import org.elasticsearch.Version; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; @@ -51,7 +49,6 @@ import static java.util.Collections.emptySet; import static org.hamcrest.Matchers.containsString; -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) public class SimpleNetty4TransportTests extends AbstractSimpleTransportTestCase { public static MockTransportService nettyFromThreadPool(Settings settings, ThreadPool threadPool, final Version version, diff --git a/plugins/examples/rest-handler/src/test/java/org/elasticsearch/example/resthandler/ExampleFixtureIT.java b/plugins/examples/rest-handler/src/test/java/org/elasticsearch/example/resthandler/ExampleFixtureIT.java index 97fc6b241ea5a..522e67b512d04 100644 --- a/plugins/examples/rest-handler/src/test/java/org/elasticsearch/example/resthandler/ExampleFixtureIT.java +++ b/plugins/examples/rest-handler/src/test/java/org/elasticsearch/example/resthandler/ExampleFixtureIT.java @@ -23,25 +23,41 @@ import org.elasticsearch.test.ESTestCase; import java.io.BufferedReader; +import java.io.BufferedWriter; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.net.InetAddress; import java.net.Socket; import java.net.URL; import java.nio.charset.StandardCharsets; -import java.util.Objects; +import java.util.ArrayList; +import java.util.List; + +import static org.hamcrest.Matchers.hasItems; public class ExampleFixtureIT extends ESTestCase { public void testExample() throws Exception { - final String stringAddress = Objects.requireNonNull(System.getProperty("external.address")); - final URL url = new URL("http://" + stringAddress); + final String externalAddress = System.getProperty("external.address"); + assertNotNull("External address must not be null", externalAddress); + final URL url = new URL("http://" + externalAddress); final InetAddress address = InetAddress.getByName(url.getHost()); try ( Socket socket = new MockSocket(address, url.getPort()); + BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), StandardCharsets.UTF_8)); BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8)) ) { - assertEquals("TEST", reader.readLine()); + writer.write("GET / HTTP/1.1\r\n"); + writer.write("Host: elastic.co\r\n\r\n"); + writer.flush(); + + final List lines = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + lines.add(line); + } + assertThat(lines, hasItems("HTTP/1.1 200 OK", "TEST")); } } } diff --git a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java index ebd8241e710ea..2f74c00ef92e2 100644 --- a/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java +++ b/plugins/repository-azure/qa/microsoft-azure-storage/src/test/java/org/elasticsearch/repositories/azure/AzureStorageFixture.java @@ -24,6 +24,8 @@ import org.elasticsearch.common.SuppressForbidden; import org.elasticsearch.common.io.Streams; import org.elasticsearch.mocksocket.MockHttpServer; +import org.elasticsearch.repositories.azure.AzureStorageTestServer.Response; +import org.elasticsearch.rest.RestStatus; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -39,6 +41,8 @@ import java.util.List; import java.util.Map; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; @@ -121,7 +125,16 @@ public void handle(HttpExchange exchange) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(exchange.getRequestBody(), out); - final AzureStorageTestServer.Response response = server.handle(method, path, query, headers, out.toByteArray()); + Response response = null; + + final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent"); + if (userAgent != null && userAgent.startsWith("Apache Ant")) { + // This is a request made by the AntFixture, just reply "OK" + response = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8)); + } else { + // Otherwise simulate a S3 response + response = server.handle(method, path, query, headers, out.toByteArray()); + } Map> responseHeaders = exchange.getResponseHeaders(); responseHeaders.put("Content-Type", singletonList(response.contentType)); diff --git a/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java b/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java index 31c85d35f3fe8..6175e581e4fd0 100644 --- a/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java +++ b/plugins/repository-gcs/qa/google-cloud-storage/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageFixture.java @@ -25,6 +25,7 @@ import org.elasticsearch.core.internal.io.Streams; import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.repositories.gcs.GoogleCloudStorageTestServer.Response; +import org.elasticsearch.rest.RestStatus; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -40,6 +41,8 @@ import java.util.List; import java.util.Map; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; @@ -123,7 +126,16 @@ public void handle(HttpExchange exchange) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(exchange.getRequestBody(), out); - final Response storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray()); + Response storageResponse = null; + + final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent"); + if (userAgent != null && userAgent.startsWith("Apache Ant")) { + // This is a request made by the AntFixture, just reply "OK" + storageResponse = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8)); + } else { + // Otherwise simulate a S3 response + storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray()); + } Map> responseHeaders = exchange.getResponseHeaders(); responseHeaders.put("Content-Type", singletonList(storageResponse.contentType)); diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 3c94f4ace7759..304e0f4ae0e1f 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -116,6 +116,11 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', dependsOn project.configurations.hdfsFixture executable = new File(project.runtimeJavaHome, 'bin/java') env 'CLASSPATH', "${ -> project.configurations.hdfsFixture.asPath }" + waitCondition = { fixture, ant -> + // the hdfs.MiniHDFS fixture writes the ports file when + // it's ready, so we can just wait for the file to exist + return fixture.portsFile.exists() + } final List miniHDFSArgs = [] diff --git a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index c8321e83d1390..cf123f85d98a9 100644 --- a/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/qa/amazon-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.io.Streams; import org.elasticsearch.mocksocket.MockHttpServer; import org.elasticsearch.repositories.s3.AmazonS3TestServer.Response; +import org.elasticsearch.rest.RestStatus; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -40,6 +41,8 @@ import java.util.List; import java.util.Map; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; @@ -122,7 +125,16 @@ public void handle(HttpExchange exchange) throws IOException { ByteArrayOutputStream out = new ByteArrayOutputStream(); Streams.copy(exchange.getRequestBody(), out); - final Response storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray()); + Response storageResponse = null; + + final String userAgent = exchange.getRequestHeaders().getFirst("User-Agent"); + if (userAgent != null && userAgent.startsWith("Apache Ant")) { + // This is a request made by the AntFixture, just reply "OK" + storageResponse = new Response(RestStatus.OK, emptyMap(), "text/plain; charset=utf-8", "OK".getBytes(UTF_8)); + } else { + // Otherwise simulate a S3 response + storageResponse = storageServer.handle(method, path, query, headers, out.toByteArray()); + } Map> responseHeaders = exchange.getResponseHeaders(); responseHeaders.put("Content-Type", singletonList(storageResponse.contentType)); diff --git a/plugins/transport-nio/build.gradle b/plugins/transport-nio/build.gradle index e76008ae28400..856a8552b8e5a 100644 --- a/plugins/transport-nio/build.gradle +++ b/plugins/transport-nio/build.gradle @@ -29,13 +29,13 @@ dependencies { compile "org.elasticsearch:elasticsearch-nio:${version}" // network stack - compile "io.netty:netty-buffer:4.1.25.Final" - compile "io.netty:netty-codec:4.1.25.Final" - compile "io.netty:netty-codec-http:4.1.25.Final" - compile "io.netty:netty-common:4.1.25.Final" - compile "io.netty:netty-handler:4.1.25.Final" - compile "io.netty:netty-resolver:4.1.25.Final" - compile "io.netty:netty-transport:4.1.25.Final" + compile "io.netty:netty-buffer:4.1.16.Final" + compile "io.netty:netty-codec:4.1.16.Final" + compile "io.netty:netty-codec-http:4.1.16.Final" + compile "io.netty:netty-common:4.1.16.Final" + compile "io.netty:netty-handler:4.1.16.Final" + compile "io.netty:netty-resolver:4.1.16.Final" + compile "io.netty:netty-transport:4.1.16.Final" } dependencyLicenses { @@ -140,6 +140,6 @@ thirdPartyAudit.excludes = [ 'org.conscrypt.AllocatedBuffer', 'org.conscrypt.BufferAllocator', - 'org.conscrypt.Conscrypt', + 'org.conscrypt.Conscrypt$Engines', 'org.conscrypt.HandshakeListener' ] \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..c546222971985 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-buffer-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +63b5fa95c74785e16f2c30ce268bc222e35c8cb5 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.25.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.25.Final.jar.sha1 deleted file mode 100644 index 3ca0cbb45ec31..0000000000000 --- a/plugins/transport-nio/licenses/netty-buffer-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f366d0cc87b158ca064d27507127e3cc4eb2f089 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..1e6c241ea0b17 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +d84a1f21768b7309c2954521cf5a1f46c2309eb1 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.25.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.25.Final.jar.sha1 deleted file mode 100644 index 5e2bc85c548dd..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3e465c75bead40d06b5b9c0612b37cf77c548887 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..71c33af1c5fc2 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +d64312378b438dfdad84267c599a053327c6f02a \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.25.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.25.Final.jar.sha1 deleted file mode 100644 index 58cb7fd987949..0000000000000 --- a/plugins/transport-nio/licenses/netty-codec-http-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -70888d3f2a829541378f68503ddd52c3193df35a \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..3edf5fcea59b3 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-common-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +177a6b30cca92f6f5f9873c9befd681377a4c328 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-common-4.1.25.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.25.Final.jar.sha1 deleted file mode 100644 index 62f85f8965513..0000000000000 --- a/plugins/transport-nio/licenses/netty-common-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e17d5c05c101fe14536ce3fb34b36c54e04791f6 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..cba27387268d1 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-handler-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +fec0e63e7dd7f4eeef7ea8dc47a1ff32dfc7ebc2 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.25.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.25.Final.jar.sha1 deleted file mode 100644 index 5391f625a4df0..0000000000000 --- a/plugins/transport-nio/licenses/netty-handler-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ecdfb8fe93a8b75db3ea8746d3437eed845c24bd \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..3571d2ecfdc48 --- /dev/null +++ b/plugins/transport-nio/licenses/netty-resolver-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +f6eb553b53fb3a90a8ac1170697093fed82eae28 \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.25.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.25.Final.jar.sha1 deleted file mode 100644 index 8225fb799e3ff..0000000000000 --- a/plugins/transport-nio/licenses/netty-resolver-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -dc0965d00746b782b33f419b005cbc130973030d \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1 new file mode 100644 index 0000000000000..e502d4c77084c --- /dev/null +++ b/plugins/transport-nio/licenses/netty-transport-4.1.16.Final.jar.sha1 @@ -0,0 +1 @@ +3c8ee2c4d4a1cbb947a5c184c7aeb2204260958b \ No newline at end of file diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.25.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.25.Final.jar.sha1 deleted file mode 100644 index 1049ea4b98bc6..0000000000000 --- a/plugins/transport-nio/licenses/netty-transport-4.1.25.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -19a6f1f649894b6705aa9d8cbcced188dff133b0 \ No newline at end of file diff --git a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpPipeliningHandler.java b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpPipeliningHandler.java index 2b702042ba7a8..1eb63364f995a 100644 --- a/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpPipeliningHandler.java +++ b/plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpPipeliningHandler.java @@ -22,13 +22,11 @@ import io.netty.channel.ChannelDuplexHandler; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPromise; -import io.netty.handler.codec.http.LastHttpContent; +import io.netty.handler.codec.http.FullHttpRequest; import org.apache.logging.log4j.Logger; import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.http.HttpPipelinedRequest; import org.elasticsearch.http.HttpPipeliningAggregator; -import org.elasticsearch.http.nio.NettyListener; -import org.elasticsearch.http.nio.NioHttpResponse; import java.nio.channels.ClosedChannelException; import java.util.List; @@ -55,17 +53,14 @@ public NioHttpPipeliningHandler(Logger logger, final int maxEventsHeld) { @Override public void channelRead(final ChannelHandlerContext ctx, final Object msg) { - if (msg instanceof LastHttpContent) { - HttpPipelinedRequest pipelinedRequest = aggregator.read(((LastHttpContent) msg)); - ctx.fireChannelRead(pipelinedRequest); - } else { - ctx.fireChannelRead(msg); - } + assert msg instanceof FullHttpRequest : "Invalid message type: " + msg.getClass(); + HttpPipelinedRequest pipelinedRequest = aggregator.read(((FullHttpRequest) msg)); + ctx.fireChannelRead(pipelinedRequest); } @Override public void write(final ChannelHandlerContext ctx, final Object msg, final ChannelPromise promise) { - assert msg instanceof NioHttpResponse : "Message must be type: " + NioHttpResponse.class; + assert msg instanceof NioHttpResponse : "Invalid message type: " + msg.getClass(); NioHttpResponse response = (NioHttpResponse) msg; boolean success = false; try { diff --git a/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy b/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy index dcd193a779b09..8c8fe7c327412 100644 --- a/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/transport-nio/src/main/plugin-metadata/plugin-security.policy @@ -23,8 +23,6 @@ grant codeBase "${codebase.elasticsearch-nio}" { }; grant codeBase "${codebase.netty-common}" { - permission java.lang.RuntimePermission "setContextClassLoader"; - // This should only currently be required as we use the netty http client for tests // netty makes and accepts socket connections permission java.net.SocketPermission "*", "accept,connect"; diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioIntegTestCase.java b/plugins/transport-nio/src/test/java/org/elasticsearch/NioIntegTestCase.java similarity index 97% rename from plugins/transport-nio/src/test/java/org/elasticsearch/test/NioIntegTestCase.java rename to plugins/transport-nio/src/test/java/org/elasticsearch/NioIntegTestCase.java index fb0ea8d9caa9f..703f7acbf8257 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioIntegTestCase.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/NioIntegTestCase.java @@ -16,12 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.test; +package org.elasticsearch; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.http.nio.NioHttpServerTransport; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.transport.nio.NioTransport; import org.elasticsearch.transport.nio.NioTransportPlugin; diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java index f045de3d566a4..6ad53521ee12a 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java @@ -46,7 +46,7 @@ import org.elasticsearch.nio.SocketChannelContext; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.NioHttpTestCase; +import org.elasticsearch.test.ESTestCase; import org.junit.Before; import org.mockito.ArgumentCaptor; @@ -69,7 +69,7 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -public class HttpReadWriteHandlerTests extends NioHttpTestCase { +public class HttpReadWriteHandlerTests extends ESTestCase { private HttpReadWriteHandler handler; private NioSocketChannel nioSocketChannel; diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NettyAdaptorTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NettyAdaptorTests.java index 5900af7f98bd0..d6944a5f510e2 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NettyAdaptorTests.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NettyAdaptorTests.java @@ -27,14 +27,14 @@ import io.netty.channel.ChannelPromise; import io.netty.channel.SimpleChannelInboundHandler; import org.elasticsearch.nio.FlushOperation; -import org.elasticsearch.test.NioHttpTestCase; +import org.elasticsearch.test.ESTestCase; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.Locale; import java.util.concurrent.atomic.AtomicBoolean; -public class NettyAdaptorTests extends NioHttpTestCase { +public class NettyAdaptorTests extends ESTestCase { public void testBasicRead() { TenIntsToStringsHandler handler = new TenIntsToStringsHandler(); diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpPipeliningHandlerTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpPipeliningHandlerTests.java index b156ef027c086..94d7db171a563 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpPipeliningHandlerTests.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpPipeliningHandlerTests.java @@ -28,17 +28,15 @@ import io.netty.channel.embedded.EmbeddedChannel; import io.netty.handler.codec.http.DefaultFullHttpRequest; import io.netty.handler.codec.http.DefaultFullHttpResponse; -import io.netty.handler.codec.http.DefaultHttpRequest; import io.netty.handler.codec.http.FullHttpRequest; import io.netty.handler.codec.http.FullHttpResponse; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpRequest; -import io.netty.handler.codec.http.HttpVersion; import io.netty.handler.codec.http.LastHttpContent; import io.netty.handler.codec.http.QueryStringDecoder; import org.elasticsearch.common.Randomness; import org.elasticsearch.http.HttpPipelinedRequest; -import org.elasticsearch.test.NioHttpTestCase; +import org.elasticsearch.test.ESTestCase; import org.junit.After; import java.nio.channels.ClosedChannelException; @@ -61,7 +59,7 @@ import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; import static org.hamcrest.core.Is.is; -public class NioHttpPipeliningHandlerTests extends NioHttpTestCase { +public class NioHttpPipeliningHandlerTests extends ESTestCase { private final ExecutorService handlerService = Executors.newFixedThreadPool(randomIntBetween(4, 8)); private final ExecutorService eventLoopService = Executors.newFixedThreadPool(1); @@ -147,38 +145,6 @@ public void testThatPipeliningWorksWhenSlowRequestsInDifferentOrder() throws Int assertTrue(embeddedChannel.isOpen()); } - public void testThatPipeliningWorksWithChunkedRequests() throws InterruptedException { - final int numberOfRequests = randomIntBetween(2, 128); - final EmbeddedChannel embeddedChannel = - new EmbeddedChannel( - new AggregateUrisAndHeadersHandler(), - new NioHttpPipeliningHandler(logger, numberOfRequests), - new WorkEmulatorHandler()); - - for (int i = 0; i < numberOfRequests; i++) { - final DefaultHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/" + i); - embeddedChannel.writeInbound(request); - embeddedChannel.writeInbound(LastHttpContent.EMPTY_LAST_CONTENT); - } - - final List latches = new ArrayList<>(); - for (int i = numberOfRequests - 1; i >= 0; i--) { - latches.add(finishRequest(Integer.toString(i))); - } - - for (final CountDownLatch latch : latches) { - latch.await(); - } - - embeddedChannel.flush(); - - for (int i = 0; i < numberOfRequests; i++) { - assertReadHttpMessageHasContent(embeddedChannel, Integer.toString(i)); - } - - assertTrue(embeddedChannel.isOpen()); - } - public void testThatPipeliningClosesConnectionWithTooManyEvents() throws InterruptedException { final int numberOfRequests = randomIntBetween(2, 128); final EmbeddedChannel embeddedChannel = new EmbeddedChannel(new NioHttpPipeliningHandler(logger, numberOfRequests), diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java index 421e235baaea0..c43fc7d072360 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioHttpServerTransportTests.java @@ -51,7 +51,7 @@ import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestChannel; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.test.NioHttpTestCase; +import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.TestThreadPool; import org.elasticsearch.threadpool.ThreadPool; import org.junit.After; @@ -83,7 +83,7 @@ /** * Tests for the {@link NioHttpServerTransport} class. */ -public class NioHttpServerTransportTests extends NioHttpTestCase { +public class NioHttpServerTransportTests extends ESTestCase { private NetworkService networkService; private ThreadPool threadPool; diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java index 4245031aa48ff..074aafd6eab4b 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/NioPipeliningIT.java @@ -20,11 +20,11 @@ package org.elasticsearch.http.nio; import io.netty.handler.codec.http.FullHttpResponse; +import org.elasticsearch.NioIntegTestCase; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.http.HttpServerTransport; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.NioHttpIntegTestCase; import java.util.Collection; import java.util.Locale; @@ -33,7 +33,7 @@ import static org.hamcrest.Matchers.is; @ClusterScope(scope = Scope.TEST, supportsDedicatedMasters = false, numDataNodes = 1) -public class NioPipeliningIT extends NioHttpIntegTestCase { +public class NioPipeliningIT extends NioIntegTestCase { @Override protected boolean addMockHttpTransport() { diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioHttpIntegTestCase.java b/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioHttpIntegTestCase.java deleted file mode 100644 index d20ac8737fadf..0000000000000 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioHttpIntegTestCase.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; - -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) -public abstract class NioHttpIntegTestCase extends NioIntegTestCase { -} diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioHttpTestCase.java b/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioHttpTestCase.java deleted file mode 100644 index 9e8a55170f9b7..0000000000000 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/test/NioHttpTestCase.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; - -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) -public abstract class NioHttpTestCase extends ESTestCase { -} diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/test/ObjectCleanerThreadThreadFilter.java b/plugins/transport-nio/src/test/java/org/elasticsearch/test/ObjectCleanerThreadThreadFilter.java deleted file mode 100644 index e47c536665d13..0000000000000 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/test/ObjectCleanerThreadThreadFilter.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.test; - -import com.carrotsearch.randomizedtesting.ThreadFilter; - -/** - * The Netty object cleaner thread is not closeable and it does not terminate in a timely manner. This means that thread leak control in - * tests will fail test suites when the object cleaner thread has not terminated. Since there is not a reliable way to terminate this thread - * we instead filter it out of thread leak control. - */ -public class ObjectCleanerThreadThreadFilter implements ThreadFilter { - - @Override - public boolean reject(final Thread t) { - // TODO: replace with constant from Netty when https://github.com/netty/netty/pull/8014 is integrated - return "ObjectCleanerThread".equals(t.getName()); - } - -} - diff --git a/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/NioTransportIT.java b/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/NioTransportIT.java index d1beed213a4e5..df53a4d79c7ad 100644 --- a/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/NioTransportIT.java +++ b/plugins/transport-nio/src/test/java/org/elasticsearch/transport/nio/NioTransportIT.java @@ -19,6 +19,7 @@ package org.elasticsearch.transport.nio; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.NioIntegTestCase; import org.elasticsearch.Version; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.client.Client; @@ -35,7 +36,6 @@ import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.ESIntegTestCase.Scope; -import org.elasticsearch.test.NioIntegTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TcpChannel; import org.elasticsearch.transport.TcpTransport; diff --git a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java index 5406c5dfde423..8f32271948763 100644 --- a/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java +++ b/qa/smoke-test-client/src/test/java/org/elasticsearch/smoketest/ESSmokeClientTestCase.java @@ -19,8 +19,6 @@ package org.elasticsearch.smoketest; -import com.carrotsearch.randomizedtesting.ThreadFilter; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -64,24 +62,8 @@ * then run JUnit. If you changed the default port, set "-Dtests.cluster=localhost:PORT" when running your test. */ @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") -@ThreadLeakFilters(filters = {ESSmokeClientTestCase.ObjectCleanerThreadThreadFilter.class}) public abstract class ESSmokeClientTestCase extends LuceneTestCase { - /** - * The Netty object cleaner thread is not closeable and it does not terminate in a timely manner. This means that thread leak control in - * tests will fail test suites when the object cleaner thread has not terminated. Since there is not a reliable way to terminate this - * thread we instead filter it out of thread leak control. - */ - public static class ObjectCleanerThreadThreadFilter implements ThreadFilter { - - @Override - public boolean reject(final Thread t) { - // TODO: replace with constant from Netty when https://github.com/netty/netty/pull/8014 is integrated - return "ObjectCleanerThread".equals(t.getName()); - } - - } - /** * Key used to eventually switch to using an external cluster and provide its transport addresses */ diff --git a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java index 4177a72e08baf..bac5423e751ea 100644 --- a/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java +++ b/qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java @@ -18,13 +18,10 @@ */ package org.elasticsearch.http; -import com.carrotsearch.randomizedtesting.ThreadFilter; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.common.network.NetworkModule; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.transport.MockTcpTransportPlugin; import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.transport.nio.MockNioTransportPlugin; @@ -34,19 +31,8 @@ import java.util.Arrays; import java.util.Collection; -@ThreadLeakFilters(filters = {HttpSmokeTestCase.ObjectCleanerThreadThreadFilter.class}) public abstract class HttpSmokeTestCase extends ESIntegTestCase { - public static class ObjectCleanerThreadThreadFilter implements ThreadFilter { - - @Override - public boolean reject(final Thread t) { - // TODO: replace with constant from Netty when https://github.com/netty/netty/pull/8014 is integrated - return "ObjectCleanerThread".equals(t.getName()); - } - - } - private static String nodeTransportTypeKey; private static String nodeHttpTypeKey; private static String clientTypeKey; diff --git a/qa/vagrant/src/test/resources/packaging/utils/utils.bash b/qa/vagrant/src/test/resources/packaging/utils/utils.bash index f8c3005de6c38..53662ca9d3c1d 100644 --- a/qa/vagrant/src/test/resources/packaging/utils/utils.bash +++ b/qa/vagrant/src/test/resources/packaging/utils/utils.bash @@ -463,8 +463,9 @@ debug_collect_logs() { set_debug_logging() { if [ "$ESCONFIG" ] && [ -d "$ESCONFIG" ] && [ -f /etc/os-release ] && (grep -qi suse /etc/os-release); then - echo 'logger.org.elasticsearch.indices: DEBUG' >> "$ESCONFIG/elasticsearch.yml" + echo 'logger.org.elasticsearch.indices: TRACE' >> "$ESCONFIG/elasticsearch.yml" echo 'logger.org.elasticsearch.gateway: TRACE' >> "$ESCONFIG/elasticsearch.yml" + echo 'logger.org.elasticsearch.cluster: DEBUG' >> "$ESCONFIG/elasticsearch.yml" fi } diff --git a/server/src/main/java/org/elasticsearch/Version.java b/server/src/main/java/org/elasticsearch/Version.java index 5111348d363bd..911614e00134d 100644 --- a/server/src/main/java/org/elasticsearch/Version.java +++ b/server/src/main/java/org/elasticsearch/Version.java @@ -120,6 +120,8 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_5_6_9 = new Version(V_5_6_9_ID, org.apache.lucene.util.Version.LUCENE_6_6_1); public static final int V_5_6_10_ID = 5061099; public static final Version V_5_6_10 = new Version(V_5_6_10_ID, org.apache.lucene.util.Version.LUCENE_6_6_1); + public static final int V_5_6_11_ID = 5061199; + public static final Version V_5_6_11 = new Version(V_5_6_11_ID, org.apache.lucene.util.Version.LUCENE_6_6_1); public static final int V_6_0_0_alpha1_ID = 6000001; public static final Version V_6_0_0_alpha1 = new Version(V_6_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_0_0); @@ -166,10 +168,10 @@ public class Version implements Comparable, ToXContentFragment { public static final Version V_6_2_3 = new Version(V_6_2_3_ID, LUCENE_7_2_1); public static final int V_6_2_4_ID = 6020499; public static final Version V_6_2_4 = new Version(V_6_2_4_ID, LUCENE_7_2_1); - public static final int V_6_2_5_ID = 6020599; - public static final Version V_6_2_5 = new Version(V_6_2_5_ID, LUCENE_7_2_1); public static final int V_6_3_0_ID = 6030099; public static final Version V_6_3_0 = new Version(V_6_3_0_ID, org.apache.lucene.util.Version.LUCENE_7_3_1); + public static final int V_6_3_1_ID = 6030199; + public static final Version V_6_3_1 = new Version(V_6_3_1_ID, org.apache.lucene.util.Version.LUCENE_7_3_1); public static final int V_6_4_0_ID = 6040099; public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0); public static final int V_7_0_0_alpha1_ID = 7000001; @@ -192,10 +194,10 @@ public static Version fromId(int id) { return V_7_0_0_alpha1; case V_6_4_0_ID: return V_6_4_0; + case V_6_3_1_ID: + return V_6_3_1; case V_6_3_0_ID: return V_6_3_0; - case V_6_2_5_ID: - return V_6_2_5; case V_6_2_4_ID: return V_6_2_4; case V_6_2_3_ID: @@ -232,6 +234,8 @@ public static Version fromId(int id) { return V_6_0_0_alpha2; case V_6_0_0_alpha1_ID: return V_6_0_0_alpha1; + case V_5_6_11_ID: + return V_5_6_11; case V_5_6_10_ID: return V_5_6_10; case V_5_6_9_ID: diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java index 59a291888d09e..0b9bcbf11b9a1 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/health/ClusterHealthRequest.java @@ -48,9 +48,9 @@ public class ClusterHealthRequest extends MasterNodeReadRequest new ParameterizedMessage("Clear SC failed on node[{}]", node), e); + /* + * We have to set the failure marker before we count down otherwise we can expose the failure marker before we have set it to a + * racing thread successfully freeing a context. This would lead to that thread responding that the clear scroll succeeded. + */ + hasFailed.set(true); if (expectedOps.countDown()) { listener.onResponse(new ClearScrollResponse(false, freedSearchContexts.get())); - } else { - hasFailed.set(true); } } } diff --git a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java index 52c3be8789416..ad3b2efd42f1c 100644 --- a/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java @@ -50,7 +50,6 @@ import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -351,8 +350,7 @@ private boolean shouldPreFilterSearchShards(SearchRequest searchRequest, GroupSh static GroupShardsIterator mergeShardsIterators(GroupShardsIterator localShardsIterator, OriginalIndices localIndices, List remoteShardIterators) { - List shards = new ArrayList<>(); - shards.addAll(remoteShardIterators); + List shards = new ArrayList<>(remoteShardIterators); for (ShardIterator shardIterator : localShardsIterator) { shards.add(new SearchShardIterator(null, shardIterator.shardId(), shardIterator.getShardRoutings(), localIndices)); } @@ -384,7 +382,7 @@ private AbstractSearchAsyncAction searchAsyncAction(SearchTask task, SearchReque clusterStateVersion, aliasFilter, concreteIndexBoosts, indexRoutings, listener, false, clusters); return new SearchPhase(action.getName()) { @Override - public void run() throws IOException { + public void run() { action.start(); } }; diff --git a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java index 22edbfca2dc12..00b7f4e6186a9 100644 --- a/server/src/main/java/org/elasticsearch/action/support/TransportAction.java +++ b/server/src/main/java/org/elasticsearch/action/support/TransportAction.java @@ -55,12 +55,6 @@ protected TransportAction(Settings settings, String actionName, ThreadPool threa this.taskManager = taskManager; } - public final ActionFuture execute(Request request) { - PlainActionFuture future = newFuture(); - execute(request, future); - return future; - } - /** * Use this method when the transport action call should result in creation of a new task associated with the call. * diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java index ce5ad12a53d6a..38766c08e08a3 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java @@ -82,8 +82,10 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request Settings.Builder settingsForOpenIndices = Settings.builder(); final Set skippedSettings = new HashSet<>(); - indexScopedSettings.validate(normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false /* don't validate wildcards */), - false); //don't validate dependencies here we check it below never allow to change the number of shards + indexScopedSettings.validate( + normalizedSettings.filter(s -> Regex.isSimpleMatchPattern(s) == false), // don't validate wildcards + false, // don't validate dependencies here we check it below never allow to change the number of shards + true); // validate internal index settings for (String key : normalizedSettings.keySet()) { Setting setting = indexScopedSettings.get(key); boolean isWildcard = setting == null && Regex.isSimpleMatchPattern(key); diff --git a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java index e8bb946c8a795..eb4e294642417 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/AbstractScopedSettings.java @@ -282,6 +282,18 @@ public final void validate(final Settings settings, final boolean validateDepend validate(settings, validateDependencies, false, false); } + /** + * Validates that all settings are registered and valid. + * + * @param settings the settings to validate + * @param validateDependencies true if dependent settings should be validated + * @param validateInternalIndex true if internal index settings should be validated + * @see Setting#getSettingsDependencies(String) + */ + public final void validate(final Settings settings, final boolean validateDependencies, final boolean validateInternalIndex) { + validate(settings, validateDependencies, false, false, validateInternalIndex); + } + /** * Validates that all settings are registered and valid. * @@ -296,6 +308,25 @@ public final void validate( final boolean validateDependencies, final boolean ignorePrivateSettings, final boolean ignoreArchivedSettings) { + validate(settings, validateDependencies, ignorePrivateSettings, ignoreArchivedSettings, false); + } + + /** + * Validates that all settings are registered and valid. + * + * @param settings the settings + * @param validateDependencies true if dependent settings should be validated + * @param ignorePrivateSettings true if private settings should be ignored during validation + * @param ignoreArchivedSettings true if archived settings should be ignored during validation + * @param validateInternalIndex true if index internal settings should be validated + * @see Setting#getSettingsDependencies(String) + */ + public final void validate( + final Settings settings, + final boolean validateDependencies, + final boolean ignorePrivateSettings, + final boolean ignoreArchivedSettings, + final boolean validateInternalIndex) { final List exceptions = new ArrayList<>(); for (final String key : settings.keySet()) { // settings iterate in deterministic fashion if (isPrivateSetting(key) && ignorePrivateSettings) { @@ -305,7 +336,7 @@ public final void validate( continue; } try { - validate(key, settings, validateDependencies); + validate(key, settings, validateDependencies, validateInternalIndex); } catch (final RuntimeException ex) { exceptions.add(ex); } @@ -314,9 +345,27 @@ public final void validate( } /** - * Validates that the setting is valid + * Validates that the settings is valid. + * + * @param key the key of the setting to validate + * @param settings the settings + * @param validateDependencies true if dependent settings should be validated + * @throws IllegalArgumentException if the setting is invalid */ - void validate(String key, Settings settings, boolean validateDependencies) { + void validate(final String key, final Settings settings, final boolean validateDependencies) { + validate(key, settings, validateDependencies, false); + } + + /** + * Validates that the settings is valid. + * + * @param key the key of the setting to validate + * @param settings the settings + * @param validateDependencies true if dependent settings should be validated + * @param validateInternalIndex true if internal index settings should be validated + * @throws IllegalArgumentException if the setting is invalid + */ + void validate(final String key, final Settings settings, final boolean validateDependencies, final boolean validateInternalIndex) { Setting setting = getRaw(key); if (setting == null) { LevensteinDistance ld = new LevensteinDistance(); @@ -356,6 +405,11 @@ void validate(String key, Settings settings, boolean validateDependencies) { } } } + // the only time that validateInternalIndex should be true is if this call is coming via the update settings API + if (validateInternalIndex && setting.getProperties().contains(Setting.Property.InternalIndex)) { + throw new IllegalArgumentException( + "can not update internal setting [" + setting.getKey() + "]; this setting is managed via a dedicated API"); + } } setting.get(settings); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/Setting.java b/server/src/main/java/org/elasticsearch/common/settings/Setting.java index f45f4bda9c9fe..7f3906ff5a251 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/Setting.java +++ b/server/src/main/java/org/elasticsearch/common/settings/Setting.java @@ -120,7 +120,13 @@ public enum Property { * Mark this setting as not copyable during an index resize (shrink or split). This property can only be applied to settings that * also have {@link Property#IndexScope}. */ - NotCopyableOnResize + NotCopyableOnResize, + + /** + * Indicates an index-level setting that is managed internally. Such a setting can only be added to an index on index creation but + * can not be updated via the update API. + */ + InternalIndex } private final Key key; @@ -152,14 +158,18 @@ private Setting(Key key, @Nullable Setting fallbackSetting, Function properties, final Property property) { + if (properties.contains(property) && properties.contains(Property.IndexScope) == false) { + throw new IllegalArgumentException("non-index-scoped setting [" + key + "] can not have property [" + property + "]"); + } + } + /** * Creates a new Setting instance * @param key the settings key for this setting. diff --git a/server/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java b/server/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java index 72e59675c1bfa..8d5ef1926cdd5 100644 --- a/server/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java +++ b/server/src/main/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandler.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; +import java.util.Collections; import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -35,6 +36,7 @@ public class BlockingClusterStatePublishResponseHandler { private final CountDownLatch latch; private final Set pendingNodes; + private final Set failedNodes; /** * Creates a new BlockingClusterStatePublishResponseHandler @@ -44,6 +46,7 @@ public BlockingClusterStatePublishResponseHandler(Set publishingT this.pendingNodes = ConcurrentCollections.newConcurrentSet(); this.pendingNodes.addAll(publishingToNodes); this.latch = new CountDownLatch(pendingNodes.size()); + this.failedNodes = ConcurrentCollections.newConcurrentSet(); } /** @@ -64,6 +67,8 @@ public void onResponse(DiscoveryNode node) { public void onFailure(DiscoveryNode node, Exception e) { boolean found = pendingNodes.remove(node); assert found : "node [" + node + "] already responded or failed"; + boolean added = failedNodes.add(node); + assert added : "duplicate failures for " + node; latch.countDown(); } @@ -86,4 +91,11 @@ public DiscoveryNode[] pendingNodes() { // nulls if some nodes responded in the meanwhile return pendingNodes.toArray(new DiscoveryNode[0]); } + + /** + * returns a set of nodes for which publication has failed. + */ + public Set getFailedNodes() { + return Collections.unmodifiableSet(failedNodes); + } } diff --git a/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java b/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java index 382a42141d83a..cd87a41526313 100644 --- a/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java +++ b/server/src/main/java/org/elasticsearch/discovery/zen/PublishClusterStateAction.java @@ -20,7 +20,6 @@ package org.elasticsearch.discovery.zen; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -41,6 +40,7 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.discovery.AckClusterStatePublishResponseHandler; import org.elasticsearch.discovery.BlockingClusterStatePublishResponseHandler; import org.elasticsearch.discovery.Discovery; @@ -207,6 +207,12 @@ private void innerPublish(final ClusterChangedEvent clusterChangedEvent, final S clusterState.version(), publishTimeout, pendingNodes); } } + // The failure is logged under debug when a sending failed. we now log a summary. + Set failedNodes = publishResponseHandler.getFailedNodes(); + if (failedNodes.isEmpty() == false) { + logger.warn("publishing cluster state with version [{}] failed for the following nodes: [{}]", + clusterChangedEvent.state().version(), failedNodes); + } } catch (InterruptedException e) { // ignore & restore interrupt Thread.currentThread().interrupt(); @@ -367,14 +373,14 @@ public static BytesReference serializeDiffClusterState(Diff diff, Version nodeVe protected void handleIncomingClusterStateRequest(BytesTransportRequest request, TransportChannel channel) throws IOException { Compressor compressor = CompressorFactory.compressor(request.bytes()); StreamInput in = request.bytes().streamInput(); - try { - if (compressor != null) { - in = compressor.streamInput(in); - } - in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry); - in.setVersion(request.version()); - synchronized (lastSeenClusterStateMutex) { - final ClusterState incomingState; + final ClusterState incomingState; + synchronized (lastSeenClusterStateMutex) { + try { + if (compressor != null) { + in = compressor.streamInput(in); + } + in = new NamedWriteableAwareStreamInput(in, namedWriteableRegistry); + in.setVersion(request.version()); // If true we received full cluster state - otherwise diffs if (in.readBoolean()) { incomingState = ClusterState.readFrom(in, transportService.getLocalNode()); @@ -391,14 +397,17 @@ protected void handleIncomingClusterStateRequest(BytesTransportRequest request, logger.debug("received diff for but don't have any local cluster state - requesting full state"); throw new IncompatibleClusterStateVersionException("have no local cluster state"); } - incomingClusterStateListener.onIncomingClusterState(incomingState); - lastSeenClusterState = incomingState; + } catch (IncompatibleClusterStateVersionException e) { + incompatibleClusterStateDiffReceivedCount.incrementAndGet(); + throw e; + } catch (Exception e) { + logger.warn("unexpected error while deserializing an incoming cluster state", e); + throw e; + } finally { + IOUtils.close(in); } - } catch (IncompatibleClusterStateVersionException e) { - incompatibleClusterStateDiffReceivedCount.incrementAndGet(); - throw e; - } finally { - IOUtils.close(in); + incomingClusterStateListener.onIncomingClusterState(incomingState); + lastSeenClusterState = incomingState; } channel.sendResponse(TransportResponse.Empty.INSTANCE); } diff --git a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java index 04253dfabb18a..4e498d393e2e1 100644 --- a/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/TransportNodesListGatewayMetaState.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.FailedNodeException; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.nodes.BaseNodeRequest; import org.elasticsearch.action.support.nodes.BaseNodeResponse; import org.elasticsearch.action.support.nodes.BaseNodesRequest; @@ -65,7 +66,9 @@ public TransportNodesListGatewayMetaState(Settings settings, ThreadPool threadPo } public ActionFuture list(String[] nodesIds, @Nullable TimeValue timeout) { - return execute(new Request(nodesIds).timeout(timeout)); + PlainActionFuture future = PlainActionFuture.newFuture(); + execute(new Request(nodesIds).timeout(timeout), future); + return future; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java index eedff2c349ce4..3fc5794798d11 100644 --- a/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java +++ b/server/src/main/java/org/elasticsearch/index/analysis/PreBuiltAnalyzerProviderFactory.java @@ -33,6 +33,7 @@ import java.util.Collection; import java.util.List; import java.util.function.Function; +import java.util.function.Supplier; import java.util.stream.Collectors; public class PreBuiltAnalyzerProviderFactory extends PreConfiguredAnalysisComponent> implements Closeable { @@ -46,13 +47,17 @@ public class PreBuiltAnalyzerProviderFactory extends PreConfiguredAnalysisCompon PreBuiltAnalyzerProviderFactory(String name, PreBuiltAnalyzers preBuiltAnalyzer) { super(name, new PreBuiltAnalyzersDelegateCache(name, preBuiltAnalyzer)); this.create = preBuiltAnalyzer::getAnalyzer; - current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, preBuiltAnalyzer.getAnalyzer(Version.CURRENT)); + Analyzer analyzer = preBuiltAnalyzer.getAnalyzer(Version.CURRENT); + analyzer.setVersion(Version.CURRENT.luceneVersion); + current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer); } - public PreBuiltAnalyzerProviderFactory(String name, PreBuiltCacheFactory.CachingStrategy cache, Function create) { + public PreBuiltAnalyzerProviderFactory(String name, PreBuiltCacheFactory.CachingStrategy cache, Supplier create) { super(name, cache); - this.create = create; - this.current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, create.apply(Version.CURRENT)); + this.create = version -> create.get(); + Analyzer analyzer = create.get(); + analyzer.setVersion(Version.CURRENT.luceneVersion); + this.current = new PreBuiltAnalyzerProvider(name, AnalyzerScope.INDICES, analyzer); } @Override @@ -71,7 +76,9 @@ public AnalyzerProvider get(IndexSettings indexSettings, @Override protected AnalyzerProvider create(Version version) { assert Version.CURRENT.equals(version) == false; - return new PreBuiltAnalyzerProvider(getName(), AnalyzerScope.INDICES, create.apply(version)); + Analyzer analyzer = create.apply(version); + analyzer.setVersion(version.luceneVersion); + return new PreBuiltAnalyzerProvider(getName(), AnalyzerScope.INDICES, analyzer); } @Override diff --git a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java index 59869d585a5c2..82b921bd233b0 100644 --- a/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java +++ b/server/src/main/java/org/elasticsearch/transport/RemoteClusterConnection.java @@ -20,7 +20,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.store.AlreadyClosedException; -import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.SetOnce; import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; @@ -40,6 +39,7 @@ import org.elasticsearch.common.util.CancellableThreads; import org.elasticsearch.common.util.concurrent.AbstractRunnable; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.threadpool.ThreadPool; import java.io.Closeable; @@ -50,7 +50,6 @@ import java.util.HashSet; import java.util.Iterator; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -61,7 +60,6 @@ import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Predicate; -import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -181,7 +179,7 @@ public void ensureConnected(ActionListener voidActionListener) { private void fetchShardsInternal(ClusterSearchShardsRequest searchShardsRequest, final ActionListener listener) { - final DiscoveryNode node = connectedNodes.get(); + final DiscoveryNode node = connectedNodes.getAny(); transportService.sendRequest(node, ClusterSearchShardsAction.NAME, searchShardsRequest, new TransportResponseHandler() { @@ -217,7 +215,7 @@ void collectNodes(ActionListener> listener) { request.clear(); request.nodes(true); request.local(true); // run this on the node that gets the request it's as good as any other - final DiscoveryNode node = connectedNodes.get(); + final DiscoveryNode node = connectedNodes.getAny(); transportService.sendRequest(node, ClusterStateAction.NAME, request, TransportRequestOptions.EMPTY, new TransportResponseHandler() { @Override @@ -255,40 +253,52 @@ public String executor() { } /** - * Returns a connection to the remote cluster. This connection might be a proxy connection that redirects internally to the - * given node. + * Returns a connection to the remote cluster, preferably a direct connection to the provided {@link DiscoveryNode}. + * If such node is not connected, the returned connection will be a proxy connection that redirects to it. */ Transport.Connection getConnection(DiscoveryNode remoteClusterNode) { - DiscoveryNode discoveryNode = connectedNodes.get(); + if (transportService.nodeConnected(remoteClusterNode)) { + return transportService.getConnection(remoteClusterNode); + } + DiscoveryNode discoveryNode = connectedNodes.getAny(); Transport.Connection connection = transportService.getConnection(discoveryNode); - return new Transport.Connection() { - @Override - public DiscoveryNode getNode() { - return remoteClusterNode; - } + return new ProxyConnection(connection, remoteClusterNode); + } - @Override - public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + static final class ProxyConnection implements Transport.Connection { + private final Transport.Connection proxyConnection; + private final DiscoveryNode targetNode; + + private ProxyConnection(Transport.Connection proxyConnection, DiscoveryNode targetNode) { + this.proxyConnection = proxyConnection; + this.targetNode = targetNode; + } + + @Override + public DiscoveryNode getNode() { + return targetNode; + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException, TransportException { - connection.sendRequest(requestId, TransportActionProxy.getProxyAction(action), - TransportActionProxy.wrapRequest(remoteClusterNode, request), options); - } + proxyConnection.sendRequest(requestId, TransportActionProxy.getProxyAction(action), + TransportActionProxy.wrapRequest(targetNode, request), options); + } - @Override - public void close() throws IOException { - assert false: "proxy connections must not be closed"; - } + @Override + public void close() { + assert false: "proxy connections must not be closed"; + } - @Override - public Version getVersion() { - return connection.getVersion(); - } - }; + @Override + public Version getVersion() { + return proxyConnection.getVersion(); + } } Transport.Connection getConnection() { - DiscoveryNode discoveryNode = connectedNodes.get(); - return transportService.getConnection(discoveryNode); + return transportService.getConnection(getAnyConnectedNode()); } @Override @@ -385,7 +395,7 @@ public void onFailure(Exception e) { } @Override - protected void doRun() throws Exception { + protected void doRun() { ActionListener listener = ActionListener.wrap((x) -> { synchronized (queue) { running.release(); @@ -590,8 +600,8 @@ boolean isNodeConnected(final DiscoveryNode node) { return connectedNodes.contains(node); } - DiscoveryNode getConnectedNode() { - return connectedNodes.get(); + DiscoveryNode getAnyConnectedNode() { + return connectedNodes.getAny(); } void addConnectedNode(DiscoveryNode node) { @@ -612,7 +622,7 @@ int getNumNodesConnected() { return connectedNodes.size(); } - private static class ConnectedNodes implements Supplier { + private static final class ConnectedNodes { private final Set nodeSet = new HashSet<>(); private final String clusterAlias; @@ -623,8 +633,7 @@ private ConnectedNodes(String clusterAlias) { this.clusterAlias = clusterAlias; } - @Override - public synchronized DiscoveryNode get() { + public synchronized DiscoveryNode getAny() { ensureIteratorAvailable(); if (currentIterator.hasNext()) { return currentIterator.next(); @@ -657,15 +666,6 @@ synchronized boolean contains(DiscoveryNode node) { return nodeSet.contains(node); } - synchronized Optional getAny() { - ensureIteratorAvailable(); - if (currentIterator.hasNext()) { - return Optional.of(currentIterator.next()); - } else { - return Optional.empty(); - } - } - private synchronized void ensureIteratorAvailable() { if (currentIterator == null) { currentIterator = nodeSet.iterator(); diff --git a/server/src/main/java/org/elasticsearch/transport/Transports.java b/server/src/main/java/org/elasticsearch/transport/Transports.java index ddd22a7fb868c..b4579374d9ec4 100644 --- a/server/src/main/java/org/elasticsearch/transport/Transports.java +++ b/server/src/main/java/org/elasticsearch/transport/Transports.java @@ -36,7 +36,7 @@ public enum Transports { * used in assertions to make sure that we do not call blocking code from * networking threads. */ - public static final boolean isTransportThread(Thread t) { + public static boolean isTransportThread(Thread t) { final String threadName = t.getName(); for (String s : Arrays.asList( HttpServerTransport.HTTP_SERVER_WORKER_THREAD_NAME_PREFIX, diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java index cac5bed4033ac..8601687b04a23 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/health/TransportClusterHealthActionTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.action.admin.cluster.health; import org.elasticsearch.Version; +import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; @@ -61,6 +62,20 @@ public void testWaitForInitializingShards() throws Exception { assertThat(TransportClusterHealthAction.prepareResponse(request, response, clusterState, null), equalTo(0)); } + public void testWaitForAllShards() { + final String[] indices = {"test"}; + final ClusterHealthRequest request = new ClusterHealthRequest(); + request.waitForActiveShards(ActiveShardCount.ALL); + + ClusterState clusterState = randomClusterStateWithInitializingShards("test", 1); + ClusterHealthResponse response = new ClusterHealthResponse("", indices, clusterState); + assertThat(TransportClusterHealthAction.prepareResponse(request, response, clusterState, null), equalTo(0)); + + clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build(); + response = new ClusterHealthResponse("", indices, clusterState); + assertThat(TransportClusterHealthAction.prepareResponse(request, response, clusterState, null), equalTo(1)); + } + ClusterState randomClusterStateWithInitializingShards(String index, final int initializingShards) { final IndexMetaData indexMetaData = IndexMetaData .builder(index) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java index 6b2e2040bca80..524d522153fe5 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/CancellableTasksTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.nodes.BaseNodeRequest; import org.elasticsearch.action.support.nodes.BaseNodesRequest; import org.elasticsearch.action.support.replication.ClusterStateCreationUtils; @@ -254,8 +255,8 @@ public void onFailure(Exception e) { request.setReason("Testing Cancellation"); request.setTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId())); // And send the cancellation request to a random node - CancelTasksResponse response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request) - .get(); + CancelTasksResponse response = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction, request); // Awaiting for the main task to finish responseLatch.await(); @@ -287,9 +288,9 @@ public void onFailure(Exception e) { } // Make sure that tasks are no longer running - ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)] - .transportListTasksAction.execute(new ListTasksRequest().setTaskId( - new TaskId(testNodes[0].getNodeId(), mainTask.getId()))).get(); + ListTasksResponse listTasksResponse = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(0, testNodes.length - 1)].transportListTasksAction, + new ListTasksRequest().setTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId()))); assertEquals(0, listTasksResponse.getTasks().size()); // Make sure that there are no leftover bans, the ban removal is async, so we might return from the cancellation @@ -326,8 +327,8 @@ public void onFailure(Exception e) { request.setReason("Testing Cancellation"); request.setParentTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId())); // And send the cancellation request to a random node - CancelTasksResponse response = testNodes[randomIntBetween(1, testNodes.length - 1)].transportCancelTasksAction.execute(request) - .get(); + CancelTasksResponse response = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(1, testNodes.length - 1)].transportCancelTasksAction, request); // Awaiting for the main task to finish responseLatch.await(); @@ -336,16 +337,11 @@ public void onFailure(Exception e) { assertThat(response.getTasks().size(), equalTo(testNodes.length)); assertBusy(() -> { - try { // Make sure that main task is no longer running - ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)] - .transportListTasksAction.execute(new ListTasksRequest().setTaskId( - new TaskId(testNodes[0].getNodeId(), mainTask.getId()))).get(); - assertEquals(0, listTasksResponse.getTasks().size()); - - } catch (ExecutionException | InterruptedException ex) { - throw new RuntimeException(ex); - } + ListTasksResponse listTasksResponse = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(0, testNodes.length - 1)].transportListTasksAction, + new ListTasksRequest().setTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId()))); + assertEquals(0, listTasksResponse.getTasks().size()); }); } @@ -378,8 +374,9 @@ public void onFailure(Exception e) { String mainNode = testNodes[0].getNodeId(); // Make sure that tasks are running - ListTasksResponse listTasksResponse = testNodes[randomIntBetween(0, testNodes.length - 1)] - .transportListTasksAction.execute(new ListTasksRequest().setParentTaskId(new TaskId(mainNode, mainTask.getId()))).get(); + ListTasksResponse listTasksResponse = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(0, testNodes.length - 1)].transportListTasksAction, + new ListTasksRequest().setParentTaskId(new TaskId(mainNode, mainTask.getId()))); assertThat(listTasksResponse.getTasks().size(), greaterThanOrEqualTo(blockOnNodes.size())); // Simulate the coordinating node leaving the cluster @@ -400,7 +397,7 @@ public void onFailure(Exception e) { request.setReason("Testing Cancellation"); request.setTaskId(new TaskId(testNodes[0].getNodeId(), mainTask.getId())); // And send the cancellation request to a random node - CancelTasksResponse response = testNodes[0].transportCancelTasksAction.execute(request).get(); + CancelTasksResponse response = ActionTestUtils.executeBlocking(testNodes[0].transportCancelTasksAction, request); logger.info("--> Done simulating issuing cancel request on the node that is about to leave the cluster"); // This node still thinks that's part of the cluster, so cancelling should look successful if (response.getTasks().size() == 0) { @@ -420,15 +417,10 @@ public void onFailure(Exception e) { assertBusy(() -> { // Make sure that tasks are no longer running - try { - ListTasksResponse listTasksResponse1 = testNodes[randomIntBetween(1, testNodes.length - 1)] - .transportListTasksAction.execute(new ListTasksRequest().setTaskId(new TaskId(mainNode, mainTask.getId()))).get(); - assertEquals(0, listTasksResponse1.getTasks().size()); - } catch (InterruptedException ex) { - Thread.currentThread().interrupt(); - } catch (ExecutionException ex2) { - fail("shouldn't be here"); - } + ListTasksResponse listTasksResponse1 = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(1, testNodes.length - 1)].transportListTasksAction, + new ListTasksRequest().setTaskId(new TaskId(mainNode, mainTask.getId()))); + assertEquals(0, listTasksResponse1.getTasks().size()); }); // Wait for clean up diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java index cb6f2b57b2bd0..27230eb518ebe 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/node/tasks/TransportTasksActionTests.java @@ -18,6 +18,7 @@ */ package org.elasticsearch.action.admin.cluster.node.tasks; +import org.elasticsearch.action.Action; import org.elasticsearch.action.ActionFuture; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.FailedNodeException; @@ -29,6 +30,7 @@ import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksResponse; import org.elasticsearch.action.admin.cluster.node.tasks.list.TaskGroup; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.action.support.nodes.BaseNodeRequest; import org.elasticsearch.action.support.nodes.BaseNodesRequest; @@ -363,7 +365,7 @@ public void onFailure(Exception e) { ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setActions("testAction*"); // pick all test actions logger.info("Listing currently running tasks using node [{}]", testNodeNum); - ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + ListTasksResponse response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); logger.info("Checking currently running tasks"); assertEquals(testNodes.length, response.getPerNodeTasks().size()); @@ -382,7 +384,7 @@ public void onFailure(Exception e) { testNode = testNodes[randomIntBetween(0, testNodes.length - 1)]; listTasksRequest = new ListTasksRequest(); listTasksRequest.setActions("testAction[n]"); // only pick node actions - response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(testNodes.length, response.getPerNodeTasks().size()); for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); @@ -396,7 +398,7 @@ public void onFailure(Exception e) { // Check task counts using transport with detailed description listTasksRequest.setDetailed(true); // same request only with detailed description - response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(testNodes.length, response.getPerNodeTasks().size()); for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); @@ -405,7 +407,7 @@ public void onFailure(Exception e) { // Make sure that the main task on coordinating node is the task that was returned to us by execute() listTasksRequest.setActions("testAction"); // only pick the main task - response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(1, response.getTasks().size()); assertEquals(mainTask.getId(), response.getTasks().get(0).getId()); @@ -433,7 +435,7 @@ public void testFindChildTasks() throws Exception { // Get the parent task ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setActions("testAction"); - ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + ListTasksResponse response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(1, response.getTasks().size()); String parentNode = response.getTasks().get(0).getTaskId().getNodeId(); long parentTaskId = response.getTasks().get(0).getId(); @@ -441,7 +443,7 @@ public void testFindChildTasks() throws Exception { // Find tasks with common parent listTasksRequest = new ListTasksRequest(); listTasksRequest.setParentTaskId(new TaskId(parentNode, parentTaskId)); - response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(testNodes.length, response.getTasks().size()); for (TaskInfo task : response.getTasks()) { assertEquals("testAction[n]", task.getAction()); @@ -467,7 +469,7 @@ public void testTaskManagementOptOut() throws Exception { // Get the parent task ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setActions("testAction*"); - ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + ListTasksResponse response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(0, response.getTasks().size()); // Release all tasks and wait for response @@ -488,7 +490,7 @@ public void testTasksDescriptions() throws Exception { TestNode testNode = testNodes[randomIntBetween(0, testNodes.length - 1)]; ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setActions("testAction[n]"); // only pick node actions - ListTasksResponse response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + ListTasksResponse response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(testNodes.length, response.getPerNodeTasks().size()); for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); @@ -498,7 +500,7 @@ public void testTasksDescriptions() throws Exception { // Check task counts using transport with detailed description long minimalDurationNanos = System.nanoTime() - maximumStartTimeNanos; listTasksRequest.setDetailed(true); // same request only with detailed description - response = testNode.transportListTasksAction.execute(listTasksRequest).get(); + response = ActionTestUtils.executeBlocking(testNode.transportListTasksAction, listTasksRequest); assertEquals(testNodes.length, response.getPerNodeTasks().size()); for (Map.Entry> entry : response.getPerNodeTasks().entrySet()) { assertEquals(1, entry.getValue().size()); @@ -536,8 +538,8 @@ public void onFailure(Exception e) { request.setNodes(testNodes[0].getNodeId()); request.setReason("Testing Cancellation"); request.setActions(actionName); - CancelTasksResponse response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request) - .get(); + CancelTasksResponse response = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction, request); // Shouldn't match any tasks since testAction doesn't support cancellation assertEquals(0, response.getTasks().size()); @@ -549,7 +551,8 @@ public void onFailure(Exception e) { request = new CancelTasksRequest(); request.setReason("Testing Cancellation"); request.setTaskId(new TaskId(testNodes[0].getNodeId(), task.getId())); - response = testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction.execute(request).get(); + response = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(0, testNodes.length - 1)].transportCancelTasksAction, request); // Shouldn't match any tasks since testAction doesn't support cancellation assertEquals(0, response.getTasks().size()); @@ -560,8 +563,8 @@ public void onFailure(Exception e) { // Make sure that task is still running ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setActions(actionName); - ListTasksResponse listResponse = testNodes[randomIntBetween(0, testNodes.length - 1)].transportListTasksAction.execute - (listTasksRequest).get(); + ListTasksResponse listResponse = ActionTestUtils.executeBlocking( + testNodes[randomIntBetween(0, testNodes.length - 1)].transportListTasksAction, listTasksRequest); assertEquals(1, listResponse.getPerNodeTasks().size()); // Verify that tasks are marked as non-cancellable for (TaskInfo taskInfo : listResponse.getTasks()) { @@ -595,7 +598,7 @@ protected NodeResponse nodeOperation(NodeRequest request) { assertEquals(0, testNode.transportService.getTaskManager().getTasks().size()); } NodesRequest request = new NodesRequest("Test Request"); - NodesResponse responses = actions[0].execute(request).get(); + NodesResponse responses = ActionTestUtils.executeBlocking(actions[0], request); assertEquals(nodesCount, responses.failureCount()); // Make sure that actions are still registered in the task manager on all nodes @@ -660,7 +663,7 @@ protected void taskOperation(TestTasksRequest request, Task task, ActionListener // should be successful on all nodes except one TestTasksRequest testTasksRequest = new TestTasksRequest(); testTasksRequest.setActions("testAction[n]"); // pick all test actions - TestTasksResponse response = tasksActions[0].execute(testTasksRequest).get(); + TestTasksResponse response = ActionTestUtils.executeBlocking(tasksActions[0], testTasksRequest); assertThat(response.getTaskFailures(), hasSize(1)); // one task failed assertThat(response.getTaskFailures().get(0).getReason(), containsString("Task level failure")); // Get successful responses from all nodes except one @@ -730,7 +733,7 @@ protected void taskOperation(TestTasksRequest request, Task task, ActionListener // should be successful on all nodes except nodes that we filtered out TestTasksRequest testTasksRequest = new TestTasksRequest(); testTasksRequest.setActions("testAction[n]"); // pick all test actions - TestTasksResponse response = tasksActions[randomIntBetween(0, nodesCount - 1)].execute(testTasksRequest).get(); + TestTasksResponse response = ActionTestUtils.executeBlocking(tasksActions[randomIntBetween(0, nodesCount - 1)], testTasksRequest); // Get successful responses from all nodes except nodes that we filtered out assertEquals(testNodes.length - filterNodes.size(), response.tasks.size()); @@ -757,7 +760,7 @@ public void testTasksToXContentGrouping() throws Exception { // Get the parent task ListTasksRequest listTasksRequest = new ListTasksRequest(); listTasksRequest.setActions(ListTasksAction.NAME + "*"); - ListTasksResponse response = testNodes[0].transportListTasksAction.execute(listTasksRequest).get(); + ListTasksResponse response = ActionTestUtils.executeBlocking(testNodes[0].transportListTasksAction, listTasksRequest); assertEquals(testNodes.length + 1, response.getTasks().size()); Map byNodes = serialize(response, true); diff --git a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java index 3c2a87e8fef00..2b83dd9aa636d 100644 --- a/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/TransportMultiSearchActionTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.action.support.TransportAction; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; @@ -141,7 +142,7 @@ protected void doExecute(SearchRequest request, ActionListener l multiSearchRequest.add(new SearchRequest()); } - MultiSearchResponse response = action.execute(multiSearchRequest).actionGet(); + MultiSearchResponse response = ActionTestUtils.executeBlocking(action, multiSearchRequest); assertThat(response.getResponses().length, equalTo(numSearchRequests)); assertThat(requests.size(), equalTo(numSearchRequests)); assertThat(errorHolder.get(), nullValue()); diff --git a/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java b/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java index d2a51070c9298..24d9633bc5154 100644 --- a/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java +++ b/server/src/test/java/org/elasticsearch/action/support/replication/BroadcastReplicationTests.java @@ -18,6 +18,8 @@ */ package org.elasticsearch.action.support.replication; +import org.elasticsearch.action.support.ActionTestUtils; +import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.NoShardAvailableActionException; @@ -120,7 +122,8 @@ public void testNotStartedPrimary() throws InterruptedException, ExecutionExcept setState(clusterService, state(index, randomBoolean(), randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.UNASSIGNED, ShardRoutingState.UNASSIGNED)); logger.debug("--> using initial state:\n{}", clusterService.state()); - Future response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index))); + PlainActionFuture response = PlainActionFuture.newFuture(); + broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index), response); for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { if (randomBoolean()) { shardRequests.v2().onFailure(new NoShardAvailableActionException(shardRequests.v1())); @@ -139,7 +142,8 @@ public void testStartedPrimary() throws InterruptedException, ExecutionException setState(clusterService, state(index, randomBoolean(), ShardRoutingState.STARTED)); logger.debug("--> using initial state:\n{}", clusterService.state()); - Future response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index))); + PlainActionFuture response = PlainActionFuture.newFuture(); + broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index), response); for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { ReplicationResponse replicationResponse = new ReplicationResponse(); replicationResponse.setShardInfo(new ReplicationResponse.ShardInfo(1, 1)); @@ -154,7 +158,8 @@ public void testResultCombine() throws InterruptedException, ExecutionException, int numShards = 1 + randomInt(3); setState(clusterService, stateWithAssignedPrimariesAndOneReplica(index, numShards)); logger.debug("--> using initial state:\n{}", clusterService.state()); - Future response = (broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index))); + PlainActionFuture response = PlainActionFuture.newFuture(); + broadcastReplicationAction.execute(new DummyBroadcastRequest().indices(index), response); int succeeded = 0; int failed = 0; for (Tuple> shardRequests : broadcastReplicationAction.capturedShardRequests) { @@ -231,17 +236,19 @@ protected void shardExecute(Task task, DummyBroadcastRequest request, ShardId sh } } - public FlushResponse assertImmediateResponse(String index, TransportFlushAction flushAction) throws InterruptedException, ExecutionException { + public FlushResponse assertImmediateResponse(String index, TransportFlushAction flushAction) { Date beginDate = new Date(); - FlushResponse flushResponse = flushAction.execute(new FlushRequest(index)).get(); + FlushResponse flushResponse = ActionTestUtils.executeBlocking(flushAction, new FlushRequest(index)); Date endDate = new Date(); long maxTime = 500; assertThat("this should not take longer than " + maxTime + " ms. The request hangs somewhere", endDate.getTime() - beginDate.getTime(), lessThanOrEqualTo(maxTime)); return flushResponse; } - public BroadcastResponse executeAndAssertImmediateResponse(TransportBroadcastReplicationAction broadcastAction, DummyBroadcastRequest request) throws InterruptedException, ExecutionException { - return (BroadcastResponse) broadcastAction.execute(request).actionGet("5s"); + public BroadcastResponse executeAndAssertImmediateResponse(TransportBroadcastReplicationAction broadcastAction, DummyBroadcastRequest request) { + PlainActionFuture response = PlainActionFuture.newFuture(); + broadcastAction.execute(request, response); + return response.actionGet("5s"); } private void assertBroadcastResponse(int total, int successful, int failed, BroadcastResponse response, Class exceptionClass) { diff --git a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java index f00768651f917..2376d5663402e 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/ScopedSettingsTests.java @@ -876,4 +876,28 @@ public void testFinalSettingUpdateFail() { Settings.builder().put(currentSettings), Settings.builder(), "node")); assertThat(exc.getMessage(), containsString("final node setting [some.final.group.foo]")); } + + public void testInternalIndexSettingsFailsValidation() { + final Setting indexInternalSetting = Setting.simpleString("index.internal", Property.InternalIndex, Property.IndexScope); + final IndexScopedSettings indexScopedSettings = + new IndexScopedSettings(Settings.EMPTY, Collections.singleton(indexInternalSetting)); + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> { + final Settings settings = Settings.builder().put("index.internal", "internal").build(); + indexScopedSettings.validate(settings, false, /* validateInternalIndex */ true); + }); + final String message = "can not update internal setting [index.internal]; this setting is managed via a dedicated API"; + assertThat(e, hasToString(containsString(message))); + } + + public void testInternalIndexSettingsSkipValidation() { + final Setting internalIndexSetting = Setting.simpleString("index.internal", Property.InternalIndex, Property.IndexScope); + final IndexScopedSettings indexScopedSettings = + new IndexScopedSettings(Settings.EMPTY, Collections.singleton(internalIndexSetting)); + // nothing should happen, validation should not throw an exception + final Settings settings = Settings.builder().put("index.internal", "internal").build(); + indexScopedSettings.validate(settings, false, /* validateInternalIndex */ false); + } + } diff --git a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java index 1ab92526e3130..c32037f44525e 100644 --- a/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java +++ b/server/src/test/java/org/elasticsearch/common/settings/SettingTests.java @@ -735,6 +735,13 @@ public void testRejectNonIndexScopedNotCopyableOnResizeSetting() { assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [NotCopyableOnResize]"))); } + public void testRejectNonIndexScopedIndexInternalSetting() { + final IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> Setting.simpleString("foo.bar", Property.InternalIndex)); + assertThat(e, hasToString(containsString("non-index-scoped setting [foo.bar] can not have property [InternalIndex]"))); + } + public void testTimeValue() { final TimeValue random = TimeValue.parseTimeValue(randomTimeValue(), "test"); diff --git a/server/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java b/server/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java index 6d0ee8a97821e..9504344236b86 100644 --- a/server/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java +++ b/server/src/test/java/org/elasticsearch/discovery/BlockingClusterStatePublishResponseHandlerTests.java @@ -85,10 +85,16 @@ public void testConcurrentAccess() throws InterruptedException { int firstRound = randomIntBetween(5, nodeCount - 1); Thread[] threads = new Thread[firstRound]; CyclicBarrier barrier = new CyclicBarrier(firstRound); + Set expectedFailures = new HashSet<>(); Set completedNodes = new HashSet<>(); for (int i = 0; i < threads.length; i++) { - completedNodes.add(allNodes[i]); - threads[i] = new Thread(new PublishResponder(randomBoolean(), allNodes[i], barrier, logger, handler)); + final DiscoveryNode node = allNodes[i]; + completedNodes.add(node); + final boolean fail = randomBoolean(); + if (fail) { + expectedFailures.add(node); + } + threads[i] = new Thread(new PublishResponder(fail, node, barrier, logger, handler)); threads[i].start(); } // wait on the threads to finish @@ -105,7 +111,12 @@ public void testConcurrentAccess() throws InterruptedException { barrier = new CyclicBarrier(secondRound); for (int i = 0; i < threads.length; i++) { - threads[i] = new Thread(new PublishResponder(randomBoolean(), allNodes[firstRound + i], barrier, logger, handler)); + final DiscoveryNode node = allNodes[firstRound + i]; + final boolean fail = randomBoolean(); + if (fail) { + expectedFailures.add(node); + } + threads[i] = new Thread(new PublishResponder(fail, node, barrier, logger, handler)); threads[i].start(); } // wait on the threads to finish @@ -114,6 +125,6 @@ public void testConcurrentAccess() throws InterruptedException { } assertTrue("expected handler not to timeout as all nodes responded", handler.awaitAllNodes(new TimeValue(10))); assertThat(handler.pendingNodes(), arrayWithSize(0)); - + assertThat(handler.getFailedNodes(), equalTo(expectedFailures)); } } diff --git a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java index 154d702e7fb77..d098c4918a76f 100644 --- a/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java +++ b/server/src/test/java/org/elasticsearch/gateway/RecoveryFromGatewayIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.action.admin.indices.recovery.RecoveryResponse; import org.elasticsearch.action.admin.indices.stats.IndexStats; import org.elasticsearch.action.admin.indices.stats.ShardStats; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.cluster.node.DiscoveryNode; @@ -552,9 +553,8 @@ public Settings onNodeStopped(String nodeName) throws Exception { DiscoveryNode node = internalCluster().getInstance(ClusterService.class, nodeName).localNode(); TransportNodesListGatewayStartedShards.NodesGatewayStartedShards response; - response = internalCluster().getInstance(TransportNodesListGatewayStartedShards.class) - .execute(new TransportNodesListGatewayStartedShards.Request(shardId, new DiscoveryNode[]{node})) - .get(); + response = ActionTestUtils.executeBlocking(internalCluster().getInstance(TransportNodesListGatewayStartedShards.class), + new TransportNodesListGatewayStartedShards.Request(shardId, new DiscoveryNode[]{node})); assertThat(response.getNodes(), hasSize(1)); assertThat(response.getNodes().get(0).allocationId(), notNullValue()); diff --git a/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java b/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java index 51c073c607e22..8093e7d38a14d 100644 --- a/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java +++ b/server/src/test/java/org/elasticsearch/indices/settings/UpdateSettingsIT.java @@ -19,19 +19,40 @@ package org.elasticsearch.indices.settings; +import org.elasticsearch.action.Action; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.master.MasterNodeRequest; +import org.elasticsearch.action.support.master.TransportMasterNodeAction; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.ClusterStateUpdateTask; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MetaData; +import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Priority; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexService; import org.elasticsearch.index.engine.VersionConflictEngineException; import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.Collections; @@ -46,6 +67,7 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertThrows; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.nullValue; public class UpdateSettingsIT extends ESIntegTestCase { @@ -79,7 +101,12 @@ public void testInvalidDynamicUpdate() { @Override protected Collection> nodePlugins() { - return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class); + return Arrays.asList(DummySettingPlugin.class, FinalSettingPlugin.class, InternalIndexSettingsPlugin.class); + } + + @Override + protected Collection> transportClientPlugins() { + return Collections.singletonList(InternalIndexSettingsPlugin.class); } public static class DummySettingPlugin extends Plugin { @@ -124,6 +151,151 @@ public List> getSettings() { } } + public static class InternalIndexSettingsPlugin extends Plugin implements ActionPlugin { + + public static final Setting INDEX_INTERNAL_SETTING = + Setting.simpleString("index.internal", Setting.Property.IndexScope, Setting.Property.InternalIndex); + + @Override + public List> getSettings() { + return Collections.singletonList(INDEX_INTERNAL_SETTING); + } + + public static class UpdateInternalIndexAction + extends Action { + + private static final UpdateInternalIndexAction INSTANCE = new UpdateInternalIndexAction(); + private static final String NAME = "indices:admin/settings/update-internal-index"; + + public UpdateInternalIndexAction() { + super(NAME); + } + + static class Request extends MasterNodeRequest { + + private String index; + private String value; + + Request() { + + } + + Request(final String index, final String value) { + this.index = index; + this.value = value; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + @Override + public void readFrom(final StreamInput in) throws IOException { + super.readFrom(in); + index = in.readString(); + value = in.readString(); + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(index); + out.writeString(value); + } + + } + + static class Response extends ActionResponse { + + } + + @Override + public Response newResponse() { + return new Response(); + } + + } + + public static class TransportUpdateInternalIndexAction + extends TransportMasterNodeAction { + + @Inject + public TransportUpdateInternalIndexAction( + final Settings settings, + final TransportService transportService, + final ClusterService clusterService, + final ThreadPool threadPool, + final ActionFilters actionFilters, + final IndexNameExpressionResolver indexNameExpressionResolver) { + super( + settings, + UpdateInternalIndexAction.NAME, + transportService, + clusterService, + threadPool, + actionFilters, + indexNameExpressionResolver, + UpdateInternalIndexAction.Request::new); + } + + @Override + protected String executor() { + return ThreadPool.Names.SAME; + } + + @Override + protected UpdateInternalIndexAction.Response newResponse() { + return new UpdateInternalIndexAction.Response(); + } + + @Override + protected void masterOperation( + final UpdateInternalIndexAction.Request request, + final ClusterState state, + final ActionListener listener) throws Exception { + clusterService.submitStateUpdateTask("update-index-internal", new ClusterStateUpdateTask() { + @Override + public ClusterState execute(final ClusterState currentState) throws Exception { + final MetaData.Builder builder = MetaData.builder(currentState.metaData()); + final IndexMetaData.Builder imdBuilder = IndexMetaData.builder(currentState.metaData().index(request.index)); + final Settings.Builder settingsBuilder = + Settings.builder() + .put(currentState.metaData().index(request.index).getSettings()) + .put("index.internal", request.value); + imdBuilder.settings(settingsBuilder); + builder.put(imdBuilder.build(), true); + return ClusterState.builder(currentState).metaData(builder).build(); + } + + @Override + public void clusterStateProcessed(final String source, final ClusterState oldState, final ClusterState newState) { + listener.onResponse(new UpdateInternalIndexAction.Response()); + } + + @Override + public void onFailure(final String source, final Exception e) { + listener.onFailure(e); + } + + }); + } + + @Override + protected ClusterBlockException checkBlock(UpdateInternalIndexAction.Request request, ClusterState state) { + return null; + } + + } + + @Override + public List> getActions() { + return Collections.singletonList( + new ActionHandler<>(UpdateInternalIndexAction.INSTANCE, TransportUpdateInternalIndexAction.class)); + } + + } + public void testUpdateDependentClusterSettings() { IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> client().admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder() @@ -474,4 +646,35 @@ public void testUpdateSettingsWithBlocks() { } } + public void testUpdateInternalIndexSettingViaSettingsAPI() { + final Settings settings = Settings.builder().put("index.internal", "internal").build(); + createIndex("test", settings); + final GetSettingsResponse response = client().admin().indices().prepareGetSettings("test").get(); + assertThat(response.getSetting("test", "index.internal"), equalTo("internal")); + // we can not update the setting via the update settings API + final IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> client().admin() + .indices() + .prepareUpdateSettings("test") + .setSettings(Settings.builder().put("index.internal", "internal-update")) + .get()); + final String message = "can not update internal setting [index.internal]; this setting is managed via a dedicated API"; + assertThat(e, hasToString(containsString(message))); + final GetSettingsResponse responseAfterAttemptedUpdate = client().admin().indices().prepareGetSettings("test").get(); + assertThat(responseAfterAttemptedUpdate.getSetting("test", "index.internal"), equalTo("internal")); + } + + public void testUpdateInternalIndexSettingViaDedicatedAPI() { + final Settings settings = Settings.builder().put("index.internal", "internal").build(); + createIndex("test", settings); + final GetSettingsResponse response = client().admin().indices().prepareGetSettings("test").get(); + assertThat(response.getSetting("test", "index.internal"), equalTo("internal")); + client().execute( + InternalIndexSettingsPlugin.UpdateInternalIndexAction.INSTANCE, + new InternalIndexSettingsPlugin.UpdateInternalIndexAction.Request("test", "internal-update")) + .actionGet(); + final GetSettingsResponse responseAfterUpdate = client().admin().indices().prepareGetSettings("test").get(); + assertThat(responseAfterUpdate.getSetting("test", "index.internal"), equalTo("internal-update")); + } + } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java index fdedf1055d66c..9417cc092d828 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/metrics/scripted/ScriptedMetricAggregatorTests.java @@ -300,6 +300,7 @@ public void testSelfReferencingAggStateAfterInit() throws IOException { } } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31307") public void testSelfReferencingAggStateAfterMap() throws IOException { try (Directory directory = newDirectory()) { Integer numDocs = randomInt(100); diff --git a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java index 0739ff5633bec..ac6f99351e46d 100644 --- a/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java +++ b/server/src/test/java/org/elasticsearch/transport/RemoteClusterConnectionTests.java @@ -81,6 +81,7 @@ import static org.hamcrest.Matchers.iterableWithSize; import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.sameInstance; import static org.hamcrest.Matchers.startsWith; public class RemoteClusterConnectionTests extends ESTestCase { @@ -992,7 +993,7 @@ public void testConnectedNodesConcurrentAccess() throws IOException, Interrupted barrier.await(); for (int j = 0; j < numGetCalls; j++) { try { - DiscoveryNode node = connection.getConnectedNode(); + DiscoveryNode node = connection.getAnyConnectedNode(); assertNotNull(node); } catch (IllegalStateException e) { if (e.getMessage().startsWith("No node available for cluster:") == false) { @@ -1053,10 +1054,10 @@ public void testClusterNameIsChecked() throws Exception { try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT, threadPool, settings); MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT, threadPool, settings); - MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, - Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build()); - MockTransportService otherClusterDiscoverable= startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, - Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) { + MockTransportService otherClusterTransport = startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, + Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build()); + MockTransportService otherClusterDiscoverable= startTransport("other_cluster_discoverable_node", otherClusterKnownNodes, + Version.CURRENT, threadPool, Settings.builder().put("cluster.name", "otherCluster").build())) { DiscoveryNode seedNode = seedTransport.getLocalDiscoNode(); DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode(); knownNodes.add(seedTransport.getLocalDiscoNode()); @@ -1093,4 +1094,76 @@ public void testClusterNameIsChecked() throws Exception { } } } + + public void testGetConnection() throws Exception { + List knownNodes = new CopyOnWriteArrayList<>(); + try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT); + MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) { + + DiscoveryNode connectedNode = seedTransport.getLocalDiscoNode(); + assertThat(connectedNode, notNullValue()); + knownNodes.add(connectedNode); + + DiscoveryNode disconnectedNode = discoverableTransport.getLocalDiscoNode(); + assertThat(disconnectedNode, notNullValue()); + knownNodes.add(disconnectedNode); + + try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) { + Transport.Connection seedConnection = new Transport.Connection() { + @Override + public DiscoveryNode getNode() { + return connectedNode; + } + + @Override + public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) + throws TransportException { + // no-op + } + + @Override + public void close() { + // no-op + } + }; + service.addDelegate(connectedNode.getAddress(), new MockTransportService.DelegateTransport(service.getOriginalTransport()) { + @Override + public Connection getConnection(DiscoveryNode node) { + if (node == connectedNode) { + return seedConnection; + } + return super.getConnection(node); + } + + @Override + public boolean nodeConnected(DiscoveryNode node) { + return node.equals(connectedNode); + } + }); + service.start(); + service.acceptIncomingRequests(); + try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", + Collections.singletonList(connectedNode), service, Integer.MAX_VALUE, n -> true)) { + connection.addConnectedNode(connectedNode); + for (int i = 0; i < 10; i++) { + //always a direct connection as the remote node is already connected + Transport.Connection remoteConnection = connection.getConnection(connectedNode); + assertSame(seedConnection, remoteConnection); + } + for (int i = 0; i < 10; i++) { + //always a direct connection as the remote node is already connected + Transport.Connection remoteConnection = connection.getConnection(service.getLocalNode()); + assertThat(remoteConnection, not(instanceOf(RemoteClusterConnection.ProxyConnection.class))); + assertThat(remoteConnection.getNode(), equalTo(service.getLocalNode())); + } + for (int i = 0; i < 10; i++) { + //always a proxy connection as the target node is not connected + Transport.Connection remoteConnection = connection.getConnection(disconnectedNode); + assertThat(remoteConnection, instanceOf(RemoteClusterConnection.ProxyConnection.class)); + assertThat(remoteConnection.getNode(), sameInstance(disconnectedNode)); + } + } + } + } + } } diff --git a/test/fixtures/example-fixture/src/main/java/example/ExampleTestFixture.java b/test/fixtures/example-fixture/src/main/java/example/ExampleTestFixture.java index 603aba1fc639b..96103d8eaa900 100644 --- a/test/fixtures/example-fixture/src/main/java/example/ExampleTestFixture.java +++ b/test/fixtures/example-fixture/src/main/java/example/ExampleTestFixture.java @@ -19,14 +19,12 @@ package example; +import com.sun.net.httpserver.HttpServer; + import java.lang.management.ManagementFactory; import java.net.Inet6Address; import java.net.InetAddress; import java.net.InetSocketAddress; -import java.nio.ByteBuffer; -import java.nio.channels.AsynchronousServerSocketChannel; -import java.nio.channels.AsynchronousSocketChannel; -import java.nio.channels.CompletionHandler; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -41,9 +39,9 @@ public static void main(String args[]) throws Exception { throw new IllegalArgumentException("ExampleTestFixture "); } Path dir = Paths.get(args[0]); - AsynchronousServerSocketChannel server = AsynchronousServerSocketChannel - .open() - .bind(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0)); + + final InetSocketAddress socketAddress = new InetSocketAddress(InetAddress.getLoopbackAddress(), 0); + final HttpServer httpServer = HttpServer.create(socketAddress, 0); // write pid file Path tmp = Files.createTempFile(dir, null, null); @@ -53,7 +51,7 @@ public static void main(String args[]) throws Exception { // write port file tmp = Files.createTempFile(dir, null, null); - InetSocketAddress bound = (InetSocketAddress) server.getLocalAddress(); + InetSocketAddress bound = httpServer.getAddress(); if (bound.getAddress() instanceof Inet6Address) { Files.write(tmp, Collections.singleton("[" + bound.getHostString() + "]:" + bound.getPort())); } else { @@ -61,21 +59,18 @@ public static void main(String args[]) throws Exception { } Files.move(tmp, dir.resolve("ports"), StandardCopyOption.ATOMIC_MOVE); + final byte[] response = "TEST\n".getBytes(StandardCharsets.UTF_8); + // go time - server.accept(null, new CompletionHandler() { - @Override - public void completed(AsynchronousSocketChannel socket, Void attachment) { - server.accept(null, this); - try (AsynchronousSocketChannel ch = socket) { - ch.write(ByteBuffer.wrap("TEST\n".getBytes(StandardCharsets.UTF_8))).get(); - } catch (Exception e) { - throw new RuntimeException(e); - } + httpServer.createContext("/", exchange -> { + try { + exchange.sendResponseHeaders(200, response.length); + exchange.getResponseBody().write(response); + } finally { + exchange.close(); } - - @Override - public void failed(Throwable exc, Void attachment) {} }); + httpServer.start(); // wait forever, until you kill me Thread.sleep(Long.MAX_VALUE); diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/test/ObjectCleanerThreadThreadFilter.java b/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java similarity index 56% rename from modules/reindex/src/test/java/org/elasticsearch/index/reindex/test/ObjectCleanerThreadThreadFilter.java rename to test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java index 948f1ec7fd6f6..cdc76292be06a 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/test/ObjectCleanerThreadThreadFilter.java +++ b/test/framework/src/main/java/org/elasticsearch/action/support/ActionTestUtils.java @@ -17,16 +17,21 @@ * under the License. */ -package org.elasticsearch.index.reindex.test; +package org.elasticsearch.action.support; -import com.carrotsearch.randomizedtesting.ThreadFilter; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; -public class ObjectCleanerThreadThreadFilter implements ThreadFilter { +import static org.elasticsearch.action.support.PlainActionFuture.newFuture; - @Override - public boolean reject(final Thread t) { - // TODO: replace with constant from Netty when https://github.com/netty/netty/pull/8014 is integrated - return "ObjectCleanerThread".equals(t.getName()); - } +public class ActionTestUtils { + + private ActionTestUtils() { /* no construction */ } + public static + Response executeBlocking(TransportAction action, Request request) { + PlainActionFuture future = newFuture(); + action.execute(request, future); + return future.actionGet(); + } } diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java index bb885d3f181b3..757fc2218d51c 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/Features.java @@ -19,8 +19,6 @@ package org.elasticsearch.test.rest.yaml; -import org.elasticsearch.test.ESIntegTestCase; - import java.util.Arrays; import java.util.List; @@ -56,9 +54,6 @@ private Features() { */ public static boolean areAllSupported(List features) { for (String feature : features) { - if ("requires_replica".equals(feature) && ESIntegTestCase.cluster().numDataNodes() >= 2) { - continue; - } if (!SUPPORTED.contains(feature)) { return false; } diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index a0af24d6cc645..5bff371d9c26a 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -16,7 +16,6 @@ buildRestTests.expectedUnconvertedCandidates = [ 'en/ml/functions/rare.asciidoc', 'en/ml/functions/sum.asciidoc', 'en/ml/functions/time.asciidoc', - 'en/ml/customurl.asciidoc', 'en/rest-api/security/ssl.asciidoc', 'en/rest-api/security/users.asciidoc', 'en/rest-api/security/tokens.asciidoc', @@ -279,6 +278,26 @@ setups['library'] = ''' {"name": "The Moon is a Harsh Mistress", "author": "Robert A. Heinlein", "release_date": "1966-04-01", "page_count": 288} ''' +setups['sample_job'] = ''' + - do: + xpack.ml.put_job: + job_id: "sample_job" + body: > + { + "description" : "Very basic job", + "analysis_config" : { + "bucket_span":"10m", + "detectors" :[ + { + "function": "count" + } + ]}, + "data_description" : { + "time_field":"timestamp", + "time_format": "epoch_ms" + } + } +''' setups['farequote_index'] = ''' - do: indices.create: diff --git a/x-pack/docs/en/ml/customurl.asciidoc b/x-pack/docs/en/ml/customurl.asciidoc index d0b7a55763180..7c773c4b9bf49 100644 --- a/x-pack/docs/en/ml/customurl.asciidoc +++ b/x-pack/docs/en/ml/customurl.asciidoc @@ -1,22 +1,53 @@ +[role="xpack"] [[ml-configuring-url]] -=== Adding Custom URLs To Machine Learning Results +=== Adding custom URLs to machine learning results When you create an advanced job or edit any job in {kib}, you can optionally -attach one or more custom URLs. You can also specify these custom settings when -you create or update jobs by using the {ml} APIs. +attach one or more custom URLs. -The custom URLs provide links from the anomalies table in the Anomaly Explorer -or Single Metric Viewer window in {kib} to custom dashboards or external -websites. For example, you can define a custom URL that provides a way for users -to drill down to the source data from the results set. +The custom URLs provide links from the anomalies table in the *Anomaly Explorer* +or *Single Metric Viewer* window in {kib} to {kib} dashboards, the *Discovery* +page, or external websites. For example, you can define a custom URL that +provides a way for users to drill down to the source data from the results set. + +When you edit a job in {kib}, it simplifies the creation of the custom URLs for +{kib} dashboards and the *Discover* page and it enables you to test your URLs. +For example: + +[role="screenshot"] +image::images/ml-customurl-edit.jpg["Edit a job to add a custom URL"] For each custom URL, you must supply the URL and a label, which is the link text -that appears in the anomalies table. +that appears in the anomalies table. You can also optionally supply a time +range. For example, these are the values that are added for `My link 1`: + +[role="screenshot"] +image::images/ml-customurl-detail.jpg["An example of a label and URL"] + +As in this case, the custom URL can contain +<>, which +are populated when you click the link in the anomalies table. In this example, +the custom URL contains `$earliest$`, `$latest$`, and `$service$` tokens, which +pass the beginning and end of the time span of the selected anomaly and the +pertinent `service` field value to the target page. If you were interested in the following anomaly, for example: [role="screenshot"] -image::images/ml-customurl.jpg["Links in the Anomaly Explorer anoamilies table"] +image::images/ml-customurl.jpg["An example of the custom URL links in the Anomaly Explorer anomalies table"] + +...clicking `My Link 1` opens the *Discover* page and shows results for the +service and date that were identified in the anomaly: + +[role="screenshot"] +image::images/ml-customurl-discover.jpg["An example of the results on the Discover page"] + +Since we specified a time range of 2 hours, the time filter restricts the +results to the time period two hours before and after the anomaly. + +You can also specify these custom URL settings when you create or update jobs by +using the {ml} APIs. [float] +[[ml-configuring-url-strings]] ==== String Substitution in Custom URLs You can use dollar sign ($) delimited tokens in a custom URL. These tokens are @@ -40,7 +71,8 @@ span of the selected anomaly to the target page. The tokens are substituted with date-time strings in ISO-8601 format. If you selected an interval of 1 hour for the anomalies table, these tokens use one hour on either side of the anomaly time as the earliest and latest times. The same is also true if the interval is -set to `Auto` and a one hour interval was chosen. +set to `Auto` and a one hour interval was chosen. You can override this behavior +by using the `time_range` setting. The `$mlcategoryregex$` and `$mlcategoryterms$` tokens pertain to jobs where you are categorizing field values. For more information about this type of analysis, @@ -55,28 +87,32 @@ the selected anomaly. Each categorization term is prefixed by a plus (+) character, so that when the token is passed to a {kib} dashboard, the resulting dashboard query seeks a match for all of the terms of the category. -For example, the following API updates a `log_categories` job to add a custom -URL that uses `$earliest$`, `$latest$`, and `$mlcategoryterms$` tokens: +For example, the following API updates a job to add a custom URL that uses +`$earliest$`, `$latest$`, and `$mlcategoryterms$` tokens: [source,js] ---------------------------------- -POST _xpack/ml/anomaly_detectors/log_categories/_update +POST _xpack/ml/anomaly_detectors/sample_job/_update { "custom_settings": { "custom_urls": [ { "url_name": "test-link1", + "time_range": "1h", "url_value": "http://localhost:5601/app/kibana#/discover?_g=(refreshInterval:(display:Off,pause:!f,value:0),time:(from:'$earliest$',mode:quick,to:'$latest$'))&_a=(columns:!(_source),index:AV3OWB68ue3Ht69t29aw,interval:auto,query:(query_string:(analyze_wildcard:!t,query:'$mlcategoryterms$')),sort:!(time,desc))" } ] } } ---------------------------------- +//CONSOLE +//TEST[setup:sample_job] When you click this custom URL in the anomalies table in {kib}, it opens up the -Discover page and displays source data for the period when the anomaly occurred. -Since this job was categorizing log messages, some `$mlcategoryterms$` token -values that were passed to the target page for an example anomaly are as follows: +*Discover* page and displays source data for the period one hour before and +after the anomaly occurred. Since this job was categorizing log messages, some +`$mlcategoryterms$` token values that were passed to the target page for an +example anomaly are as follows: [role="screenshot"] image::images/ml-categoryterms.jpg["A query for category terms on the Discover page in {kib}"] diff --git a/x-pack/docs/en/ml/images/ml-customurl-detail.jpg b/x-pack/docs/en/ml/images/ml-customurl-detail.jpg new file mode 100644 index 0000000000000..f7b2907c5564d Binary files /dev/null and b/x-pack/docs/en/ml/images/ml-customurl-detail.jpg differ diff --git a/x-pack/docs/en/ml/images/ml-customurl-discover.jpg b/x-pack/docs/en/ml/images/ml-customurl-discover.jpg new file mode 100644 index 0000000000000..991560919bd50 Binary files /dev/null and b/x-pack/docs/en/ml/images/ml-customurl-discover.jpg differ diff --git a/x-pack/docs/en/ml/images/ml-customurl-edit.jpg b/x-pack/docs/en/ml/images/ml-customurl-edit.jpg new file mode 100644 index 0000000000000..a209284c78669 Binary files /dev/null and b/x-pack/docs/en/ml/images/ml-customurl-edit.jpg differ diff --git a/x-pack/docs/en/ml/images/ml-customurl.jpg b/x-pack/docs/en/ml/images/ml-customurl.jpg index 4dce63e24cb28..e43c6d2588c21 100644 Binary files a/x-pack/docs/en/ml/images/ml-customurl.jpg and b/x-pack/docs/en/ml/images/ml-customurl.jpg differ diff --git a/x-pack/docs/en/rest-api/ml/close-job.asciidoc b/x-pack/docs/en/rest-api/ml/close-job.asciidoc index 3e612f5171da1..8e7e8eb0ce850 100644 --- a/x-pack/docs/en/rest-api/ml/close-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/close-job.asciidoc @@ -5,7 +5,7 @@ Close Jobs ++++ -This API enables you to close one or more jobs. +Closes one or more jobs. A job can be opened and closed multiple times throughout its lifecycle. A closed job cannot receive data or perform analysis diff --git a/x-pack/docs/en/rest-api/ml/delete-calendar-event.asciidoc b/x-pack/docs/en/rest-api/ml/delete-calendar-event.asciidoc index b6f3c644acfea..73458f3179197 100644 --- a/x-pack/docs/en/rest-api/ml/delete-calendar-event.asciidoc +++ b/x-pack/docs/en/rest-api/ml/delete-calendar-event.asciidoc @@ -5,7 +5,7 @@ Delete Events from Calendar ++++ -This API enables you to delete scheduled events from a calendar. +Deletes scheduled events from a calendar. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/delete-calendar-job.asciidoc b/x-pack/docs/en/rest-api/ml/delete-calendar-job.asciidoc index 54fe9ebdaba9b..94388c0c4b680 100644 --- a/x-pack/docs/en/rest-api/ml/delete-calendar-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/delete-calendar-job.asciidoc @@ -5,7 +5,7 @@ Delete Jobs from Calendar ++++ -This API enables you to delete jobs from a calendar. +Deletes jobs from a calendar. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/delete-calendar.asciidoc b/x-pack/docs/en/rest-api/ml/delete-calendar.asciidoc index 37b3ae3c87b36..f7673b545748b 100644 --- a/x-pack/docs/en/rest-api/ml/delete-calendar.asciidoc +++ b/x-pack/docs/en/rest-api/ml/delete-calendar.asciidoc @@ -5,7 +5,7 @@ Delete Calendar ++++ -This API enables you to delete a calendar. +Deletes a calendar. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/delete-datafeed.asciidoc b/x-pack/docs/en/rest-api/ml/delete-datafeed.asciidoc index de529267f4f7c..db4fd5c177aed 100644 --- a/x-pack/docs/en/rest-api/ml/delete-datafeed.asciidoc +++ b/x-pack/docs/en/rest-api/ml/delete-datafeed.asciidoc @@ -5,7 +5,7 @@ Delete {dfeeds-cap} ++++ -This API enables you to delete an existing {dfeed}. +Deletes an existing {dfeed}. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/delete-job.asciidoc b/x-pack/docs/en/rest-api/ml/delete-job.asciidoc index 7aaba59e122eb..c01b08545b638 100644 --- a/x-pack/docs/en/rest-api/ml/delete-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/delete-job.asciidoc @@ -5,7 +5,7 @@ Delete Jobs ++++ -This API enables you to delete an existing anomaly detection job. +Deletes an existing anomaly detection job. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/delete-snapshot.asciidoc b/x-pack/docs/en/rest-api/ml/delete-snapshot.asciidoc index b63e37a1b454b..2ab0116fe74d9 100644 --- a/x-pack/docs/en/rest-api/ml/delete-snapshot.asciidoc +++ b/x-pack/docs/en/rest-api/ml/delete-snapshot.asciidoc @@ -5,7 +5,7 @@ Delete Model Snapshots ++++ -This API enables you to delete an existing model snapshot. +Deletes an existing model snapshot. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/flush-job.asciidoc b/x-pack/docs/en/rest-api/ml/flush-job.asciidoc index 2a65c5284fcf4..934a2d81b1778 100644 --- a/x-pack/docs/en/rest-api/ml/flush-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/flush-job.asciidoc @@ -5,7 +5,7 @@ Flush Jobs ++++ -This API forces any buffered data to be processed by the job. +Forces any buffered data to be processed by the job. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/forecast.asciidoc b/x-pack/docs/en/rest-api/ml/forecast.asciidoc index 9e3e48a2e7b38..169debef7b6cb 100644 --- a/x-pack/docs/en/rest-api/ml/forecast.asciidoc +++ b/x-pack/docs/en/rest-api/ml/forecast.asciidoc @@ -5,8 +5,7 @@ Forecast Jobs ++++ -This API uses historical behavior to predict the future behavior of a time -series. +Predict the future behavior of a time series by using historical behavior. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-bucket.asciidoc b/x-pack/docs/en/rest-api/ml/get-bucket.asciidoc index 9a20d4fc15e52..95b05ff7f5dd2 100644 --- a/x-pack/docs/en/rest-api/ml/get-bucket.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-bucket.asciidoc @@ -5,7 +5,7 @@ Get Buckets ++++ -This API enables you to retrieve job results for one or more buckets. +Retrieves job results for one or more buckets. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-calendar-event.asciidoc b/x-pack/docs/en/rest-api/ml/get-calendar-event.asciidoc index 1a10ad68d7f22..e89173c3382d9 100644 --- a/x-pack/docs/en/rest-api/ml/get-calendar-event.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-calendar-event.asciidoc @@ -5,7 +5,7 @@ Get Scheduled Events ++++ -This API enables you to retrieve information about the scheduled events in +Retrieves information about the scheduled events in calendars. diff --git a/x-pack/docs/en/rest-api/ml/get-calendar.asciidoc b/x-pack/docs/en/rest-api/ml/get-calendar.asciidoc index 245d570947276..ae95fd9968893 100644 --- a/x-pack/docs/en/rest-api/ml/get-calendar.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-calendar.asciidoc @@ -5,7 +5,7 @@ Get Calendars ++++ -This API enables you to retrieve configuration information for calendars. +Retrieves configuration information for calendars. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-category.asciidoc b/x-pack/docs/en/rest-api/ml/get-category.asciidoc index 9e69083355bbb..13f274133c0d1 100644 --- a/x-pack/docs/en/rest-api/ml/get-category.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-category.asciidoc @@ -5,7 +5,7 @@ Get Categories ++++ -This API enables you to retrieve job results for one or more categories. +Retrieves job results for one or more categories. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-datafeed-stats.asciidoc b/x-pack/docs/en/rest-api/ml/get-datafeed-stats.asciidoc index 6c5b3af650b9a..2869e8222f86f 100644 --- a/x-pack/docs/en/rest-api/ml/get-datafeed-stats.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-datafeed-stats.asciidoc @@ -5,7 +5,7 @@ Get {dfeed-cap} Statistics ++++ -This API enables you to retrieve usage information for {dfeeds}. +Retrieves usage information for {dfeeds}. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-datafeed.asciidoc b/x-pack/docs/en/rest-api/ml/get-datafeed.asciidoc index 8d582ed672aff..0fa51773fd162 100644 --- a/x-pack/docs/en/rest-api/ml/get-datafeed.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-datafeed.asciidoc @@ -5,7 +5,7 @@ Get {dfeeds-cap} ++++ -This API enables you to retrieve configuration information for {dfeeds}. +Retrieves configuration information for {dfeeds}. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-influencer.asciidoc b/x-pack/docs/en/rest-api/ml/get-influencer.asciidoc index 6c49e66e944ac..bffd2b8e09633 100644 --- a/x-pack/docs/en/rest-api/ml/get-influencer.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-influencer.asciidoc @@ -5,7 +5,7 @@ Get Influencers ++++ -This API enables you to retrieve job results for one or more influencers. +Retrieves job results for one or more influencers. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-job-stats.asciidoc b/x-pack/docs/en/rest-api/ml/get-job-stats.asciidoc index 48ebac280aae3..bd59ee8b258fa 100644 --- a/x-pack/docs/en/rest-api/ml/get-job-stats.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-job-stats.asciidoc @@ -5,7 +5,7 @@ Get Job Statistics ++++ -This API enables you to retrieve usage information for jobs. +Retrieves usage information for jobs. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-job.asciidoc b/x-pack/docs/en/rest-api/ml/get-job.asciidoc index c606cc5ad40e2..2e95d8e01bbb7 100644 --- a/x-pack/docs/en/rest-api/ml/get-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-job.asciidoc @@ -5,7 +5,7 @@ Get Jobs ++++ -This API enables you to retrieve configuration information for jobs. +Retrieves configuration information for jobs. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-overall-buckets.asciidoc b/x-pack/docs/en/rest-api/ml/get-overall-buckets.asciidoc index d0e8c1f214bd4..f2581f4904e37 100644 --- a/x-pack/docs/en/rest-api/ml/get-overall-buckets.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-overall-buckets.asciidoc @@ -5,7 +5,7 @@ Get Overall Buckets ++++ -This API enables you to retrieve overall bucket results that summarize the +Retrieves overall bucket results that summarize the bucket results of multiple jobs. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-record.asciidoc b/x-pack/docs/en/rest-api/ml/get-record.asciidoc index 6cd222027e66b..1870b44159760 100644 --- a/x-pack/docs/en/rest-api/ml/get-record.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-record.asciidoc @@ -5,7 +5,7 @@ Get Records ++++ -This API enables you to retrieve anomaly records for a job. +Retrieves anomaly records for a job. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/get-snapshot.asciidoc b/x-pack/docs/en/rest-api/ml/get-snapshot.asciidoc index b992f5be7df31..24e82af1f199f 100644 --- a/x-pack/docs/en/rest-api/ml/get-snapshot.asciidoc +++ b/x-pack/docs/en/rest-api/ml/get-snapshot.asciidoc @@ -5,7 +5,7 @@ Get Model Snapshots ++++ -This API enables you to retrieve information about model snapshots. +Retrieves information about model snapshots. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/open-job.asciidoc b/x-pack/docs/en/rest-api/ml/open-job.asciidoc index 37d201ed2264e..59d5568ac775a 100644 --- a/x-pack/docs/en/rest-api/ml/open-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/open-job.asciidoc @@ -5,7 +5,7 @@ Open Jobs ++++ -This API enables you to open one or more jobs. +Opens one or more jobs. A job must be opened in order for it to be ready to receive and analyze data. A job can be opened and closed multiple times throughout its lifecycle. diff --git a/x-pack/docs/en/rest-api/ml/post-calendar-event.asciidoc b/x-pack/docs/en/rest-api/ml/post-calendar-event.asciidoc index ab0c1ebef64ab..41af0841d2e83 100644 --- a/x-pack/docs/en/rest-api/ml/post-calendar-event.asciidoc +++ b/x-pack/docs/en/rest-api/ml/post-calendar-event.asciidoc @@ -5,7 +5,7 @@ Add Events to Calendar ++++ -This API enables you to post scheduled events in a calendar. +Posts scheduled events in a calendar. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/post-data.asciidoc b/x-pack/docs/en/rest-api/ml/post-data.asciidoc index ec20be5dadb12..40354d7f6f760 100644 --- a/x-pack/docs/en/rest-api/ml/post-data.asciidoc +++ b/x-pack/docs/en/rest-api/ml/post-data.asciidoc @@ -5,7 +5,7 @@ Post Data to Jobs ++++ -This API enables you to send data to an anomaly detection job for analysis. +Sends data to an anomaly detection job for analysis. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/preview-datafeed.asciidoc b/x-pack/docs/en/rest-api/ml/preview-datafeed.asciidoc index 5f3bc5054e394..e6b51f8ef069f 100644 --- a/x-pack/docs/en/rest-api/ml/preview-datafeed.asciidoc +++ b/x-pack/docs/en/rest-api/ml/preview-datafeed.asciidoc @@ -5,7 +5,7 @@ Preview {dfeeds-cap} ++++ -This API enables you to preview a {dfeed}. +Previews a {dfeed}. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/put-calendar-job.asciidoc b/x-pack/docs/en/rest-api/ml/put-calendar-job.asciidoc index 5d2c012a919d7..6940957b15926 100644 --- a/x-pack/docs/en/rest-api/ml/put-calendar-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/put-calendar-job.asciidoc @@ -5,7 +5,7 @@ Add Jobs to Calendar ++++ -This API enables you to add a job to a calendar. +Adds a job to a calendar. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/put-calendar.asciidoc b/x-pack/docs/en/rest-api/ml/put-calendar.asciidoc index 23997906cb7f0..a82da5a2c0c0a 100644 --- a/x-pack/docs/en/rest-api/ml/put-calendar.asciidoc +++ b/x-pack/docs/en/rest-api/ml/put-calendar.asciidoc @@ -5,7 +5,7 @@ Create Calendar ++++ -This API enables you to instantiate a calendar. +Instantiates a calendar. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/put-datafeed.asciidoc b/x-pack/docs/en/rest-api/ml/put-datafeed.asciidoc index f1e41cad8b343..6b8ad932a1d42 100644 --- a/x-pack/docs/en/rest-api/ml/put-datafeed.asciidoc +++ b/x-pack/docs/en/rest-api/ml/put-datafeed.asciidoc @@ -5,7 +5,7 @@ Create {dfeeds-cap} ++++ -This API enables you to instantiate a {dfeed}. +Instantiates a {dfeed}. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/put-job.asciidoc b/x-pack/docs/en/rest-api/ml/put-job.asciidoc index 62d15acf05383..1c436f53d32e7 100644 --- a/x-pack/docs/en/rest-api/ml/put-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/put-job.asciidoc @@ -5,7 +5,7 @@ Create Jobs ++++ -This API enables you to instantiate a job. +Instantiates a job. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/revert-snapshot.asciidoc b/x-pack/docs/en/rest-api/ml/revert-snapshot.asciidoc index 72b934a56b79f..1dc3046ac4f22 100644 --- a/x-pack/docs/en/rest-api/ml/revert-snapshot.asciidoc +++ b/x-pack/docs/en/rest-api/ml/revert-snapshot.asciidoc @@ -5,7 +5,7 @@ Revert Model Snapshots ++++ -This API enables you to revert to a specific snapshot. +Reverts to a specific snapshot. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/start-datafeed.asciidoc b/x-pack/docs/en/rest-api/ml/start-datafeed.asciidoc index 865ca4ae99722..fa3ea35a751f7 100644 --- a/x-pack/docs/en/rest-api/ml/start-datafeed.asciidoc +++ b/x-pack/docs/en/rest-api/ml/start-datafeed.asciidoc @@ -5,7 +5,7 @@ Start {dfeeds-cap} ++++ -This API enables you to start one or more {dfeeds}. +Starts one or more {dfeeds}. A {dfeed} must be started in order to retrieve data from {es}. A {dfeed} can be started and stopped multiple times throughout its lifecycle. diff --git a/x-pack/docs/en/rest-api/ml/stop-datafeed.asciidoc b/x-pack/docs/en/rest-api/ml/stop-datafeed.asciidoc index 3511c9362c3fa..27872ff5a2080 100644 --- a/x-pack/docs/en/rest-api/ml/stop-datafeed.asciidoc +++ b/x-pack/docs/en/rest-api/ml/stop-datafeed.asciidoc @@ -5,7 +5,7 @@ Stop {dfeeds-cap} ++++ -This API enables you to stop one or more {dfeeds}. +Stops one or more {dfeeds}. A {dfeed} that is stopped ceases to retrieve data from {es}. A {dfeed} can be started and stopped multiple times throughout its lifecycle. diff --git a/x-pack/docs/en/rest-api/ml/update-datafeed.asciidoc b/x-pack/docs/en/rest-api/ml/update-datafeed.asciidoc index 277a9ce31773f..bc9462347c1c0 100644 --- a/x-pack/docs/en/rest-api/ml/update-datafeed.asciidoc +++ b/x-pack/docs/en/rest-api/ml/update-datafeed.asciidoc @@ -5,7 +5,7 @@ Update {dfeeds-cap} ++++ -This API enables you to update certain properties of a {dfeed}. +Updates certain properties of a {dfeed}. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/update-job.asciidoc b/x-pack/docs/en/rest-api/ml/update-job.asciidoc index ebb41523de01a..23046febc2e45 100644 --- a/x-pack/docs/en/rest-api/ml/update-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/update-job.asciidoc @@ -5,7 +5,7 @@ Update Jobs ++++ -This API enables you to update certain properties of a job. +Updates certain properties of a job. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/update-snapshot.asciidoc b/x-pack/docs/en/rest-api/ml/update-snapshot.asciidoc index 74a684619c411..8c98a7b732186 100644 --- a/x-pack/docs/en/rest-api/ml/update-snapshot.asciidoc +++ b/x-pack/docs/en/rest-api/ml/update-snapshot.asciidoc @@ -5,7 +5,7 @@ Update Model Snapshots ++++ -This API enables you to update certain properties of a snapshot. +Updates certain properties of a snapshot. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/validate-detector.asciidoc b/x-pack/docs/en/rest-api/ml/validate-detector.asciidoc index 6fc5fea6fbb94..f688ef91cfe53 100644 --- a/x-pack/docs/en/rest-api/ml/validate-detector.asciidoc +++ b/x-pack/docs/en/rest-api/ml/validate-detector.asciidoc @@ -5,7 +5,7 @@ Validate Detectors ++++ -This API validates detector configuration information. +Validates detector configuration information. ==== Request diff --git a/x-pack/docs/en/rest-api/ml/validate-job.asciidoc b/x-pack/docs/en/rest-api/ml/validate-job.asciidoc index b206734bc033f..61d0c70514e8d 100644 --- a/x-pack/docs/en/rest-api/ml/validate-job.asciidoc +++ b/x-pack/docs/en/rest-api/ml/validate-job.asciidoc @@ -5,7 +5,7 @@ Validate Jobs ++++ -This API validates job configuration information. +Validates job configuration information. ==== Request diff --git a/x-pack/docs/en/rest-api/rollup/delete-job.asciidoc b/x-pack/docs/en/rest-api/rollup/delete-job.asciidoc index 056a4470480a0..b795e0b28c760 100644 --- a/x-pack/docs/en/rest-api/rollup/delete-job.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/delete-job.asciidoc @@ -5,9 +5,37 @@ Delete Job ++++ +experimental[] + This API deletes an existing rollup job. The job can be started or stopped, in both cases it will be deleted. Attempting to delete a non-existing job will throw an exception +.Deleting the job does not delete rolled up data +********************************** +When a job is deleted, that only removes the process that is actively monitoring and rolling up data. +It does not delete any previously rolled up data. This is by design; a user may wish to roll up a static dataset. Because +the dataset is static, once it has been fully rolled up there is no need to keep the indexing Rollup job around (as there +will be no new data). So the job may be deleted, leaving behind the rolled up data for analysis. + +If you wish to also remove the rollup data, and the rollup index only contains the data for a single job, you can simply +delete the whole rollup index. If the rollup index stores data from several jobs, you must issue a Delete-By-Query that +targets the Rollup job's ID in the rollup index: + + +[source,js] +-------------------------------------------------- +POST my_rollup_index/_delete_by_query +{ + "query": { + "term": { + "_rollup.id": "the_rollup_job_id" + } + } +} +-------------------------------------------------- +// NOTCONSOLE + +********************************** ==== Request `DELETE _xpack/rollup/job/` diff --git a/x-pack/docs/en/rest-api/rollup/get-job.asciidoc b/x-pack/docs/en/rest-api/rollup/get-job.asciidoc index 4482a87527930..7a7db9258b88a 100644 --- a/x-pack/docs/en/rest-api/rollup/get-job.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/get-job.asciidoc @@ -5,6 +5,8 @@ Get Job ++++ +experimental[] + This API returns the configuration, stats and status of rollup jobs. The API can return the details for a single job, or for all jobs. diff --git a/x-pack/docs/en/rest-api/rollup/put-job.asciidoc b/x-pack/docs/en/rest-api/rollup/put-job.asciidoc index 2cc869e1e3467..1449acadc636d 100644 --- a/x-pack/docs/en/rest-api/rollup/put-job.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/put-job.asciidoc @@ -5,6 +5,8 @@ Create Job ++++ +experimental[] + This API enables you to create a rollup job. The job will be created in a `STOPPED` state, and must be started with the <>. diff --git a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc index 5a4dab69d937f..270ad005144ac 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-caps.asciidoc @@ -5,6 +5,8 @@ Get Rollup Caps ++++ +experimental[] + This API returns the rollup capabilities that have been configured for an index or index pattern. This API is useful because a rollup job is often configured to rollup only a subset of fields from the source index. Furthermore, only certain aggregations can be configured for various fields, leading to a limited subset of functionality depending on diff --git a/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc index 85f1a57caa763..ef0ea6f00f7ce 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-job-config.asciidoc @@ -2,6 +2,8 @@ [[rollup-job-config]] === Rollup Job Configuration +experimental[] + The Rollup Job Configuration contains all the details about how the rollup job should run, when it indexes documents, and what future queries will be able to execute against the rollup index. diff --git a/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc b/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc index 557953fefb231..470cbc4eaf57d 100644 --- a/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/rollup-search.asciidoc @@ -5,6 +5,8 @@ Rollup Search ++++ +experimental[] + The Rollup Search endpoint allows searching rolled-up data using the standard query DSL. The Rollup Search endpoint is needed because, internally, rolled-up documents utilize a different document structure than the original data. The Rollup Search endpoint rewrites standard query DSL into a format that matches the rollup documents, then takes the response diff --git a/x-pack/docs/en/rest-api/rollup/start-job.asciidoc b/x-pack/docs/en/rest-api/rollup/start-job.asciidoc index b8eccd5fbce82..9a0a0a7e4f01c 100644 --- a/x-pack/docs/en/rest-api/rollup/start-job.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/start-job.asciidoc @@ -5,6 +5,8 @@ Start Job ++++ +experimental[] + This API starts an existing, stopped rollup job. If the job does not exist an exception will be thrown. Starting an already started job has no action. diff --git a/x-pack/docs/en/rest-api/rollup/stop-job.asciidoc b/x-pack/docs/en/rest-api/rollup/stop-job.asciidoc index 9da3872a10b00..6050740270503 100644 --- a/x-pack/docs/en/rest-api/rollup/stop-job.asciidoc +++ b/x-pack/docs/en/rest-api/rollup/stop-job.asciidoc @@ -5,6 +5,8 @@ Stop Job ++++ +experimental[] + This API stops an existing, started rollup job. If the job does not exist an exception will be thrown. Stopping an already stopped job has no action. diff --git a/x-pack/docs/en/rollup/api-quickref.asciidoc b/x-pack/docs/en/rollup/api-quickref.asciidoc index 1ae6de4ee011c..10aed1b572d38 100644 --- a/x-pack/docs/en/rollup/api-quickref.asciidoc +++ b/x-pack/docs/en/rollup/api-quickref.asciidoc @@ -1,6 +1,8 @@ [[rollup-api-quickref]] == API Quick Reference +experimental[] + Most {rollup} endpoints have the following base: [source,js] diff --git a/x-pack/docs/en/rollup/index.asciidoc b/x-pack/docs/en/rollup/index.asciidoc index 69cd872e59669..9ac89341bfe99 100644 --- a/x-pack/docs/en/rollup/index.asciidoc +++ b/x-pack/docs/en/rollup/index.asciidoc @@ -18,7 +18,8 @@ for analysis, but at a fraction of the storage cost of raw data. * <> * <> * <> -* <> +* <> +* <> -- @@ -27,4 +28,5 @@ include::overview.asciidoc[] include::api-quickref.asciidoc[] include::rollup-getting-started.asciidoc[] include::understanding-groups.asciidoc[] +include::rollup-agg-limitations.asciidoc[] include::rollup-search-limitations.asciidoc[] \ No newline at end of file diff --git a/x-pack/docs/en/rollup/overview.asciidoc b/x-pack/docs/en/rollup/overview.asciidoc index cee244a2ec241..a3f29f23bd107 100644 --- a/x-pack/docs/en/rollup/overview.asciidoc +++ b/x-pack/docs/en/rollup/overview.asciidoc @@ -1,6 +1,8 @@ [[rollup-overview]] == Overview +experimental[] + Time-based data (documents that are predominantly identified by their timestamp) often have associated retention policies to manage data growth. For example, your system may be generating 500,000 documents every second. That will generate 43 million documents per day, and nearly 16 billion documents a year. diff --git a/x-pack/docs/en/rollup/rollup-agg-limitations.asciidoc b/x-pack/docs/en/rollup/rollup-agg-limitations.asciidoc new file mode 100644 index 0000000000000..cd20622d93c8d --- /dev/null +++ b/x-pack/docs/en/rollup/rollup-agg-limitations.asciidoc @@ -0,0 +1,24 @@ +[[rollup-agg-limitations]] +== Rollup Aggregation Limitations + +experimental[] + +There are some limitations to how fields can be rolled up / aggregated. This page highlights the major limitations so that +you are aware of them. + +[float] +=== Limited aggregation components + +The Rollup functionality allows fields to be grouped with the following aggregations: + +- Date Histogram aggregation +- Histogram aggregation +- Terms aggregation + +And the following metrics are allowed to be specified for numeric fields: + +- Min aggregation +- Max aggregation +- Sum aggregation +- Average aggregation +- Value Count aggregation \ No newline at end of file diff --git a/x-pack/docs/en/rollup/rollup-getting-started.asciidoc b/x-pack/docs/en/rollup/rollup-getting-started.asciidoc index cf96d67454083..24f68dddd8101 100644 --- a/x-pack/docs/en/rollup/rollup-getting-started.asciidoc +++ b/x-pack/docs/en/rollup/rollup-getting-started.asciidoc @@ -1,6 +1,8 @@ [[rollup-getting-started]] == Getting Started +experimental[] + To use the Rollup feature, you need to create one or more "Rollup Jobs". These jobs run continuously in the background and rollup the index or indices that you specify, placing the rolled documents in a secondary index (also of your choosing). diff --git a/x-pack/docs/en/rollup/rollup-search-limitations.asciidoc b/x-pack/docs/en/rollup/rollup-search-limitations.asciidoc index de47404a29da3..57ba23eebccbe 100644 --- a/x-pack/docs/en/rollup/rollup-search-limitations.asciidoc +++ b/x-pack/docs/en/rollup/rollup-search-limitations.asciidoc @@ -1,6 +1,8 @@ [[rollup-search-limitations]] == Rollup Search Limitations +experimental[] + While we feel the Rollup function is extremely flexible, the nature of summarizing data means there will be some limitations. Once live data is thrown away, you will always lose some flexibility. @@ -100,8 +102,8 @@ The Rollup functionality allows `query`'s in the search request, but with a limi - MatchAll Query - Any compound query (Boolean, Boosting, ConstantScore, etc) -Furthermore, these queries can only use fields that were also saved in the rollup job. If you wish to filter on a keyword `hostname` field, -that field must have been configured in the rollup job under a `terms` grouping. +Furthermore, these queries can only use fields that were also saved in the rollup job as a `group`. +If you wish to filter on a keyword `hostname` field, that field must have been configured in the rollup job under a `terms` grouping. If you attempt to use an unsupported query, or the query references a field that wasn't configured in the rollup job, an exception will be thrown. We expect the list of support queries to grow over time as more are implemented. diff --git a/x-pack/docs/en/rollup/understanding-groups.asciidoc b/x-pack/docs/en/rollup/understanding-groups.asciidoc index d6eef54fab87e..f57f905ae04c8 100644 --- a/x-pack/docs/en/rollup/understanding-groups.asciidoc +++ b/x-pack/docs/en/rollup/understanding-groups.asciidoc @@ -1,6 +1,8 @@ [[rollup-understanding-groups]] == Understanding Groups +experimental[] + To preserve flexibility, Rollup Jobs are defined based on how future queries may need to use the data. Traditionally, systems force the admin to make decisions about what metrics to rollup and on what interval. E.g. The average of `cpu_time` on an hourly basis. This is limiting; if, at a future date, the admin wishes to see the average of `cpu_time` on an hourly basis _and partitioned by `host_name`_, diff --git a/x-pack/docs/en/security/auditing/output-index.asciidoc b/x-pack/docs/en/security/auditing/output-index.asciidoc index a07bd7a8d06eb..1c59762ea2a98 100644 --- a/x-pack/docs/en/security/auditing/output-index.asciidoc +++ b/x-pack/docs/en/security/auditing/output-index.asciidoc @@ -36,6 +36,13 @@ xpack.security.audit.index.settings: number_of_replicas: 1 ---------------------------- +These settings apply to the local audit indices, as well as to the +<>, but only if the remote cluster +does *not* have {security} installed, or the {es} versions are different. +If the remote cluster has {security} installed, and the versions coincide, the +settings for the audit indices there will take precedence, +even if they are unspecified (i.e. left to defaults). + NOTE: Audit events are batched for indexing so there is a lag before events appear in the index. You can control how frequently batches of events are pushed to the index by setting diff --git a/x-pack/docs/en/sql/functions/index.asciidoc b/x-pack/docs/en/sql/functions/index.asciidoc index 7220c73623056..a4e7028cf39c3 100644 --- a/x-pack/docs/en/sql/functions/index.asciidoc +++ b/x-pack/docs/en/sql/functions/index.asciidoc @@ -1,9 +1,11 @@ [[sql-functions]] == Functions and Operators +{es-sql} provides a number of built-in operators and functions. + === Comparison Operators -Elasticsearch SQL supports the following comparison operators: +{es-sql} supports the following comparison operators: * Equality (`=`) @@ -12,7 +14,7 @@ Elasticsearch SQL supports the following comparison operators: include-tagged::{sql-specs}/filter.sql-spec[whereFieldEquality] -------------------------------------------------- -* Inequality (`<>` or `!=`) +* Inequality (`<>` or `!=` or `<=>`) ["source","sql",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -43,7 +45,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereIsNotNullAndIsNull] === Logical Operators -Elasticsearch SQL supports the following logical operators: +{es-sql} supports the following logical operators: * `AND` @@ -69,7 +71,7 @@ include-tagged::{sql-specs}/filter.sql-spec[whereFieldEqualityNot] === Math Operators -Elasticsearch SQL supports the following math operators: +{es-sql} supports the following math operators: * Add (`+`) @@ -106,7 +108,7 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[multiply] include-tagged::{sql-specs}/arithmetic.sql-spec[divide] -------------------------------------------------- -* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] (`%`) +* https://en.wikipedia.org/wiki/Modulo_operation[Modulo] or Reminder(`%`) ["source","sql",subs="attributes,callouts,macros"] -------------------------------------------------- @@ -115,25 +117,49 @@ include-tagged::{sql-specs}/arithmetic.sql-spec[mod] === Math Functions -==== Basic -* https://en.wikipedia.org/wiki/Absolute_value[Absolute value] (`ABS`) +All math and trigonometric functions require their input (where applicable) +to be numeric. + +==== Generic + +* `ABS` + +https://en.wikipedia.org/wiki/Absolute_value[Absolute value], returns \[same type as input] ["source","sql",subs="attributes,callouts,macros"] -------------------------------------------------- include-tagged::{sql-specs}/math.sql-spec[abs] -------------------------------------------------- -* https://en.wikipedia.org/wiki/Rounding#Round_half_up[Round] (`ROUND`) +* `CBRT` + +https://en.wikipedia.org/wiki/Cube_root[Cube root], returns `double` // TODO make the example in the tests presentable -NOTE: This rounds "half up" meaning that `ROUND(-1.5)` results in `-1`. +* `CEIL` + +https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Ceiling], returns `double` -* https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Ceiling] (`CEIL`) +* `CEILING` + +Same as `CEIL` // TODO make the example in the tests presentable +* `E` + +https://en.wikipedia.org/wiki/E_%28mathematical_constant%29[Euler's number], returns `2.7182818284590452354` + + +* https://en.wikipedia.org/wiki/Rounding#Round_half_up[Round] (`ROUND`) + +// TODO make the example in the tests presentable + +NOTE: This rounds "half up" meaning that `ROUND(-1.5)` results in `-1`. + + * https://en.wikipedia.org/wiki/Floor_and_ceiling_functions[Floor] (`FLOOR`) // TODO make the example in the tests presentable @@ -159,10 +185,6 @@ include-tagged::{sql-specs}/math.sql-spec[log10] include-tagged::{sql-specs}/math.sql-spec[sqrt] -------------------------------------------------- -* https://en.wikipedia.org/wiki/Cube_root[Cube root] (`CBRT`) - -// TODO make the example in the tests presentable - * https://en.wikipedia.org/wiki/Exponential_function[e^x^] (`EXP`) ["source","sql",subs="attributes,callouts,macros"] diff --git a/x-pack/docs/en/sql/getting-started.asciidoc b/x-pack/docs/en/sql/getting-started.asciidoc index 966ac5336cfdd..24f01910551bb 100644 --- a/x-pack/docs/en/sql/getting-started.asciidoc +++ b/x-pack/docs/en/sql/getting-started.asciidoc @@ -1,7 +1,7 @@ [[sql-getting-started]] == Getting Started with SQL -To start using Elasticsearch SQL, create +To start using {es-sql}, create an index with some data to experiment with: [source,js] diff --git a/x-pack/docs/en/sql/index.asciidoc b/x-pack/docs/en/sql/index.asciidoc index d2f9bba099384..902ea8ada7e22 100644 --- a/x-pack/docs/en/sql/index.asciidoc +++ b/x-pack/docs/en/sql/index.asciidoc @@ -6,12 +6,20 @@ :sql-specs: {sql-tests}/src/main/resources :jdbc-tests: {sql-tests}/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc :security-tests: {sql-tests}/security/src/test/java/org/elasticsearch/xpack/qa/sql/security +:es-sql: Elasticsearch SQL [partintro] -- + +experimental[] + X-Pack includes a SQL feature to execute SQL against Elasticsearch -indices and return tabular results. There are four main components: +indices and return results in tabular format. +<>:: + Overview of {es-sql} and its features. +<>:: + Start using SQL right away in {es} <>:: Accepts SQL in a JSON document, executes it, and returns the results. @@ -19,12 +27,19 @@ indices and return tabular results. There are four main components: Accepts SQL in a JSON document and translates it into a native Elasticsearch query and returns that. <>:: - Command line application that connects to Elasticsearch to execute + Command-line application that connects to {es} to execute SQL and print tabular results. <>:: - A JDBC driver for Elasticsearch. + A JDBC driver for {es}. +<>:: + List of functions and operators supported. +<>:: + Overview of the {es-sql} language, such as data types, syntax and + reserved keywords. + -- +include::overview.asciidoc[] include::getting-started.asciidoc[] include::endpoints/index.asciidoc[] include::functions/index.asciidoc[] diff --git a/x-pack/docs/en/sql/language/data-types.asciidoc b/x-pack/docs/en/sql/language/data-types.asciidoc index f97299eb0f243..322269bddaf8f 100644 --- a/x-pack/docs/en/sql/language/data-types.asciidoc +++ b/x-pack/docs/en/sql/language/data-types.asciidoc @@ -1,6 +1,87 @@ [[sql-data-types]] -=== Data Type and Mapping +=== Data Types + +Most of {es} <> are available in {es-sql}, as indicated below: + +[cols="^,^,^",options="header"] + +|=== +| {es} type | SQL type | SQL precision + +3+h| Core types + +| <> | `null` | 0 +| <> | `boolean` | 1 +| <> | `tinyint` | 3 +| <> | `smallint` | 5 +| <> | `integer` | 10 +| <> | `long` | 19 +| <> | `double` | 15 +| <> | `real` | 7 +| <> | `float` | 16 +| <> | `float` | 19 +| <> | `varchar` | based on <> +| <> | `varchar` | 2,147,483,647 +| <> | `varbinary` | 2,147,483,647 +| <> | `timestamp` | 24 + +3+h| Complex types + +| <> | `struct` | 0 +| <> | `struct` | 0 + +3+h| Unsupported types + +| _types not mentioned above_ | `unsupported`| 0 + +|=== + +Obviously, not all types in {es} have an equivalent in SQL and vice-versa hence why, {es-sql} +uses the data type _particularities_ of the former over the latter as ultimately {es} is the backing store. + + +[[sql-multi-field]] +[float] +==== SQL and multi-fields + +A core concept in {es} is that of an `analyzed` field, that is a full-text value that is interpreted in order +to be effectively indexed. These fields are of type <> and are not used for sorting or aggregations as their actual value depends on the <> used hence why {es} also offers the <> type for storing the _exact_ +value. + +In most case, and the default actually, is to use both types when for strings which {es} supports through <>, that is the ability to index the same string in multiple ways; for example index it both as `text` for search but also as `keyword` for sorting and aggregations. + +As SQL requires exact values, when encountering a `text` field {es-sql} will search for an exact multi-field that it can use for comparisons, sorting and aggregations. +To do that, it will search for the first `keyword` that it can find that is _not_ normalized and use that as the original field _exact_ value. + +Consider the following `string` mapping: + +[source, js] +---- +{ + "first_name" : { + "type" : "text", + "fields" : { + "raw" : { + "type" : "keyword" + } + } + } +} +---- + +The following SQL query: + +[source, sql] +---- +SELECT first_name FROM index WHERE first_name = 'John' +---- + +is identical to: + +[source, sql] +---- +SELECT first_name FROM index WHERE first_name.raw = 'John' +---- + +as {es-sql} automatically _picks_ up the `raw` multi-field from `raw` for exact matching. -// TODO finish this -List of data types in SQL and how they actually map to Elasticsearch. -Also mention the corner cases - multi-fields, names with dots, etc... diff --git a/x-pack/docs/en/sql/overview.asciidoc b/x-pack/docs/en/sql/overview.asciidoc new file mode 100644 index 0000000000000..34d0dfb538352 --- /dev/null +++ b/x-pack/docs/en/sql/overview.asciidoc @@ -0,0 +1,30 @@ +[[sql-overview]] +== Overview + +{es-sql} aims to provide a powerful yet lightweight SQL interface to {es}. + +[[sql-introduction]] +=== Introduction + +{es-sql} is an X-Pack component that allows SQL-like queries to be executed in real-time against {es}. +Whether using the REST interface, command-line or JDBC, any client can use SQL to search and aggregate data +_natively_ inside {es}. +One can think of {es-sql} as a _translator_, one that understands both SQL and {es} and makes it easy to read and process data in real-time, at scale by leveraging {es} capabilities. + +[[sql-why]] +=== Why {es-sql} ? + +Native integration:: + +{es-sql} is built from the ground up for {es}. Each and every query is efficiently executed against the relevant nodes according to the underlying storage. + +No external parts:: + +No need for additional hardware, processes, runtimes or libraries to query {es}; {es-sql} eliminates extra moving parts by running _inside_ the {es} cluster. + +Lightweight and efficient:: + +{es-sql} does not abstract {es} and its search capabilities - on the contrary, it embrases and exposes to SQL to allow proper full-text search, in real-time, in the same declarative, succint fashion. + + + diff --git a/x-pack/docs/en/sql/standalone.asciidoc b/x-pack/docs/en/sql/standalone.asciidoc deleted file mode 100644 index 6649ce9eb8de9..0000000000000 --- a/x-pack/docs/en/sql/standalone.asciidoc +++ /dev/null @@ -1,9 +0,0 @@ -[[elasticsearch-sql-standalone]] -= Elasticsearch SQL Standalone - -:es-repo-dir: {docdir}/../../../../../elasticsearch/docs - -:edit_url: -include::{es-repo-dir}/reference/index-shared3.asciidoc[] -:edit_url!: -include::index.asciidoc[] diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java index 68e1201816dc4..79e569987fa02 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEvent.java @@ -16,7 +16,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.MlMetaIndex; -import org.elasticsearch.xpack.core.ml.job.config.Connective; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Operator; import org.elasticsearch.xpack.core.ml.job.config.RuleAction; @@ -148,8 +147,7 @@ public DetectionRule toDetectionRule(TimeValue bucketSpan) { conditions.add(RuleCondition.createTime(Operator.LT, bucketEndTime)); DetectionRule.Builder builder = new DetectionRule.Builder(conditions); - builder.setActions(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING); - builder.setConditionsConnective(Connective.AND); + builder.setActions(RuleAction.SKIP_RESULT, RuleAction.SKIP_MODEL_UPDATE); return builder.build(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Condition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Condition.java deleted file mode 100644 index 7d3074df0ae28..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Condition.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.ml.job.config; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser.ValueType; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.job.messages.Messages; -import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; - -import java.io.IOException; -import java.util.Objects; -import java.util.regex.Pattern; -import java.util.regex.PatternSyntaxException; - -/** - * A class that describes a condition. - * The {@linkplain Operator} enum defines the available - * comparisons a condition can use. - */ -public class Condition implements ToXContentObject, Writeable { - public static final ParseField CONDITION_FIELD = new ParseField("condition"); - public static final ParseField VALUE_FIELD = new ParseField("value"); - - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - CONDITION_FIELD.getPreferredName(), a -> new Condition((Operator) a[0], (String) a[1])); - - static { - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return Operator.fromString(p.text()); - } - throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, Operator.OPERATOR_FIELD, ValueType.STRING); - PARSER.declareField(ConstructingObjectParser.constructorArg(), p -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return p.text(); - } - if (p.currentToken() == XContentParser.Token.VALUE_NULL) { - return null; - } - throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, VALUE_FIELD, ValueType.STRING_OR_NULL); - } - - private final Operator op; - private final String value; - - public Condition(StreamInput in) throws IOException { - op = Operator.readFromStream(in); - value = in.readOptionalString(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - op.writeTo(out); - out.writeOptionalString(value); - } - - public Condition(Operator op, String value) { - if (value == null) { - throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NULL)); - } - - if (op.expectsANumericArgument()) { - try { - Double.parseDouble(value); - } catch (NumberFormatException nfe) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, value); - throw ExceptionsHelper.badRequestException(msg); - } - } else { - try { - Pattern.compile(value); - } catch (PatternSyntaxException e) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX, value); - throw ExceptionsHelper.badRequestException(msg); - } - } - this.op = op; - this.value = value; - } - - public Operator getOperator() { - return op; - } - - public String getValue() { - return value; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field(Operator.OPERATOR_FIELD.getPreferredName(), op); - builder.field(VALUE_FIELD.getPreferredName(), value); - builder.endObject(); - return builder; - } - - @Override - public int hashCode() { - return Objects.hash(op, value); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - - if (getClass() != obj.getClass()) { - return false; - } - - Condition other = (Condition) obj; - return Objects.equals(this.op, other.op) && - Objects.equals(this.value, other.value); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Connective.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Connective.java deleted file mode 100644 index 0b4ad010fdd32..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Connective.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.ml.job.config; - -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; -import java.util.Locale; - -public enum Connective implements Writeable { - OR, AND; - - /** - * Case-insensitive from string method. - * - * @param value - * String representation - * @return The connective type - */ - public static Connective fromString(String value) { - return Connective.valueOf(value.toUpperCase(Locale.ROOT)); - } - - public static Connective readFromStream(StreamInput in) throws IOException { - return in.readEnum(Connective.class); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeEnum(this); - } - - @Override - public String toString() { - return name().toLowerCase(Locale.ROOT); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java index 0948e978c886e..fbdb2f6662a77 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRule.java @@ -6,22 +6,18 @@ package org.elasticsearch.xpack.core.ml.job.config; import org.elasticsearch.Version; -import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.MlParserType; import org.elasticsearch.xpack.core.ml.job.messages.Messages; import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; @@ -30,16 +26,15 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.stream.Collectors; public class DetectionRule implements ToXContentObject, Writeable { + public static final Version VERSION_INTRODUCED = Version.V_6_4_0; + public static final ParseField DETECTION_RULE_FIELD = new ParseField("detection_rule"); - public static final ParseField ACTIONS_FIELD = new ParseField("actions", "rule_action"); - public static final ParseField TARGET_FIELD_NAME_FIELD = new ParseField("target_field_name"); - public static final ParseField TARGET_FIELD_VALUE_FIELD = new ParseField("target_field_value"); - public static final ParseField CONDITIONS_CONNECTIVE_FIELD = new ParseField("conditions_connective"); - public static final ParseField CONDITIONS_FIELD = new ParseField("conditions", "rule_conditions"); + public static final ParseField ACTIONS_FIELD = new ParseField("actions"); + public static final ParseField SCOPE_FIELD = new ParseField("scope"); + public static final ParseField CONDITIONS_FIELD = new ParseField("conditions"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ObjectParser METADATA_PARSER = @@ -55,87 +50,44 @@ public class DetectionRule implements ToXContentObject, Writeable { ObjectParser parser = PARSERS.get(parserType); assert parser != null; parser.declareStringArray(Builder::setActions, ACTIONS_FIELD); - parser.declareString(Builder::setTargetFieldName, TARGET_FIELD_NAME_FIELD); - parser.declareString(Builder::setTargetFieldValue, TARGET_FIELD_VALUE_FIELD); - parser.declareField(Builder::setConditionsConnective, p -> { - if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return Connective.fromString(p.text()); - } - throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, CONDITIONS_CONNECTIVE_FIELD, ValueType.STRING); + parser.declareObject(Builder::setScope, RuleScope.parser(parserType), SCOPE_FIELD); parser.declareObjectArray(Builder::setConditions, (p, c) -> RuleCondition.PARSERS.get(parserType).apply(p, c), CONDITIONS_FIELD); } } private final EnumSet actions; - private final String targetFieldName; - private final String targetFieldValue; - private final Connective conditionsConnective; + private final RuleScope scope; private final List conditions; - private DetectionRule(EnumSet actions, @Nullable String targetFieldName, @Nullable String targetFieldValue, - Connective conditionsConnective, List conditions) { + private DetectionRule(EnumSet actions, RuleScope scope, List conditions) { this.actions = Objects.requireNonNull(actions); - this.targetFieldName = targetFieldName; - this.targetFieldValue = targetFieldValue; - this.conditionsConnective = Objects.requireNonNull(conditionsConnective); + this.scope = Objects.requireNonNull(scope); this.conditions = Collections.unmodifiableList(conditions); } public DetectionRule(StreamInput in) throws IOException { - actions = EnumSet.noneOf(RuleAction.class); - if (in.getVersion().before(Version.V_6_2_0)) { - actions.add(RuleAction.readFromStream(in)); - } else { - int actionsCount = in.readVInt(); - for (int i = 0; i < actionsCount; ++i) { - actions.add(RuleAction.readFromStream(in)); - } - } - - conditionsConnective = Connective.readFromStream(in); - int size = in.readVInt(); - conditions = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - conditions.add(new RuleCondition(in)); - } - targetFieldName = in.readOptionalString(); - targetFieldValue = in.readOptionalString(); + actions = in.readEnumSet(RuleAction.class); + scope = new RuleScope(in); + conditions = in.readList(RuleCondition::new); } @Override public void writeTo(StreamOutput out) throws IOException { - if (out.getVersion().before(Version.V_6_2_0)) { - // Only filter_results is supported prior to 6.2.0 - RuleAction.FILTER_RESULTS.writeTo(out); - } else { - out.writeVInt(actions.size()); - for (RuleAction action : actions) { - action.writeTo(out); - } - } - - conditionsConnective.writeTo(out); - out.writeVInt(conditions.size()); - for (RuleCondition condition : conditions) { - condition.writeTo(out); - } - out.writeOptionalString(targetFieldName); - out.writeOptionalString(targetFieldValue); + out.writeEnumSet(actions); + scope.writeTo(out); + out.writeList(conditions); } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ACTIONS_FIELD.getPreferredName(), actions); - builder.field(CONDITIONS_CONNECTIVE_FIELD.getPreferredName(), conditionsConnective); - builder.field(CONDITIONS_FIELD.getPreferredName(), conditions); - if (targetFieldName != null) { - builder.field(TARGET_FIELD_NAME_FIELD.getPreferredName(), targetFieldName); + if (scope.isEmpty() == false) { + builder.field(SCOPE_FIELD.getPreferredName(), scope); } - if (targetFieldValue != null) { - builder.field(TARGET_FIELD_VALUE_FIELD.getPreferredName(), targetFieldValue); + if (conditions.isEmpty() == false) { + builder.field(CONDITIONS_FIELD.getPreferredName(), conditions); } builder.endObject(); return builder; @@ -145,18 +97,8 @@ public EnumSet getActions() { return actions; } - @Nullable - public String getTargetFieldName() { - return targetFieldName; - } - - @Nullable - public String getTargetFieldValue() { - return targetFieldValue; - } - - public Connective getConditionsConnective() { - return conditionsConnective; + public RuleScope getScope() { + return scope; } public List getConditions() { @@ -164,7 +106,7 @@ public List getConditions() { } public Set extractReferencedFilters() { - return conditions.stream().map(RuleCondition::getFilterId).filter(Objects::nonNull).collect(Collectors.toSet()); + return scope.getReferencedFilters(); } @Override @@ -179,29 +121,29 @@ public boolean equals(Object obj) { DetectionRule other = (DetectionRule) obj; return Objects.equals(actions, other.actions) - && Objects.equals(targetFieldName, other.targetFieldName) - && Objects.equals(targetFieldValue, other.targetFieldValue) - && Objects.equals(conditionsConnective, other.conditionsConnective) + && Objects.equals(scope, other.scope) && Objects.equals(conditions, other.conditions); } @Override public int hashCode() { - return Objects.hash(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions); + return Objects.hash(actions, scope, conditions); } public static class Builder { - private EnumSet actions = EnumSet.of(RuleAction.FILTER_RESULTS); - private String targetFieldName; - private String targetFieldValue; - private Connective conditionsConnective = Connective.OR; + private EnumSet actions = EnumSet.of(RuleAction.SKIP_RESULT); + private RuleScope scope = new RuleScope(); private List conditions = Collections.emptyList(); + public Builder(RuleScope.Builder scope) { + this.scope = scope.build(); + } + public Builder(List conditions) { this.conditions = ExceptionsHelper.requireNonNull(conditions, CONDITIONS_FIELD.getPreferredName()); } - private Builder() { + Builder() { } public Builder setActions(List actions) { @@ -221,18 +163,8 @@ public Builder setActions(RuleAction... actions) { return this; } - public Builder setTargetFieldName(String targetFieldName) { - this.targetFieldName = targetFieldName; - return this; - } - - public Builder setTargetFieldValue(String targetFieldValue) { - this.targetFieldValue = targetFieldValue; - return this; - } - - public Builder setConditionsConnective(Connective connective) { - this.conditionsConnective = ExceptionsHelper.requireNonNull(connective, CONDITIONS_CONNECTIVE_FIELD.getPreferredName()); + public Builder setScope(RuleScope scope) { + this.scope = Objects.requireNonNull(scope); return this; } @@ -242,22 +174,11 @@ public Builder setConditions(List conditions) { } public DetectionRule build() { - if (targetFieldValue != null && targetFieldName == null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME, targetFieldValue); + if (scope.isEmpty() && conditions.isEmpty()) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_SCOPE_OR_CONDITION); throw ExceptionsHelper.badRequestException(msg); } - if (conditions == null || conditions.isEmpty()) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION); - throw ExceptionsHelper.badRequestException(msg); - } - for (RuleCondition condition : conditions) { - if (condition.getType().isCategorical() && targetFieldName != null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, - DetectionRule.TARGET_FIELD_NAME_FIELD.getPreferredName()); - throw ExceptionsHelper.badRequestException(msg); - } - } - return new DetectionRule(actions, targetFieldName, targetFieldValue, conditionsConnective, conditions); + return new DetectionRule(actions, scope, conditions); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java index e5cf4b16f6e73..bae5e654ba4fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeSet; import java.util.stream.Collectors; @@ -84,8 +85,7 @@ public String toString() { public static final ParseField PARTITION_FIELD_NAME_FIELD = new ParseField("partition_field_name"); public static final ParseField USE_NULL_FIELD = new ParseField("use_null"); public static final ParseField EXCLUDE_FREQUENT_FIELD = new ParseField("exclude_frequent"); - // TODO: Remove the deprecated detector_rules setting in 7.0 - public static final ParseField RULES_FIELD = new ParseField("rules", "detector_rules"); + public static final ParseField CUSTOM_RULES_FIELD = new ParseField("custom_rules"); public static final ParseField DETECTOR_INDEX = new ParseField("detector_index"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly @@ -113,7 +113,7 @@ public String toString() { throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); }, EXCLUDE_FREQUENT_FIELD, ObjectParser.ValueType.STRING); parser.declareObjectArray(Builder::setRules, (p, c) -> - DetectionRule.PARSERS.get(parserType).apply(p, c).build(), RULES_FIELD); + DetectionRule.PARSERS.get(parserType).apply(p, c).build(), CUSTOM_RULES_FIELD); parser.declareInt(Builder::setDetectorIndex, DETECTOR_INDEX); } } @@ -209,6 +209,19 @@ public String toString() { DetectorFunction.TIME_OF_WEEK ); + /** + * Functions that do not support rule conditions: + *

    + *
  • lat_long - because it is a multivariate feature + *
  • metric - because having the same conditions on min,max,mean is + * error-prone + *
  • rare - because the actual/typical value is not something a user can anticipate + *
  • freq_rare - because the actual/typical value is not something a user can anticipate + *
+ */ + static final EnumSet FUNCTIONS_WITHOUT_RULE_CONDITION_SUPPORT = EnumSet.of( + DetectorFunction.LAT_LONG, DetectorFunction.METRIC, DetectorFunction.RARE, DetectorFunction.FREQ_RARE); + /** * field names cannot contain any of these characters * ", \ @@ -263,7 +276,11 @@ public void writeTo(StreamOutput out) throws IOException { } else { out.writeBoolean(false); } - out.writeList(rules); + if (out.getVersion().onOrAfter(DetectionRule.VERSION_INTRODUCED)) { + out.writeList(rules); + } else { + out.writeList(Collections.emptyList()); + } if (out.getVersion().onOrAfter(Version.V_5_5_0)) { out.writeInt(detectorIndex); } @@ -293,7 +310,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(EXCLUDE_FREQUENT_FIELD.getPreferredName(), excludeFrequent); } if (rules.isEmpty() == false) { - builder.field(RULES_FIELD.getPreferredName(), rules); + builder.field(CUSTOM_RULES_FIELD.getPreferredName(), rules); } // negative means "unknown", which should only happen for a 5.4 job if (detectorIndex >= 0 @@ -467,17 +484,6 @@ public int hashCode() { public static class Builder { - /** - * Functions that do not support rules: - *
    - *
  • lat_long - because it is a multivariate feature - *
  • metric - because having the same conditions on min,max,mean is - * error-prone - *
- */ - static final EnumSet FUNCTIONS_WITHOUT_RULE_SUPPORT = EnumSet.of( - DetectorFunction.LAT_LONG, DetectorFunction.METRIC); - private String detectorDescription; private DetectorFunction function; private String fieldName; @@ -598,14 +604,8 @@ public Detector build() { } DetectorFunction function = this.function == null ? DetectorFunction.METRIC : this.function; - if (rules.isEmpty() == false) { - if (FUNCTIONS_WITHOUT_RULE_SUPPORT.contains(function)) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION, function); - throw ExceptionsHelper.badRequestException(msg); - } - for (DetectionRule rule : rules) { - checkScoping(rule); - } + for (DetectionRule rule : rules) { + validateRule(rule, function); } // partition, by and over field names cannot be duplicates @@ -691,96 +691,37 @@ private static boolean containsInvalidChar(String field) { return field.chars().anyMatch(Character::isISOControl); } - private void checkScoping(DetectionRule rule) throws ElasticsearchParseException { - String targetFieldName = rule.getTargetFieldName(); - checkTargetFieldNameIsValid(extractAnalysisFields(), targetFieldName); - for (RuleCondition condition : rule.getConditions()) { - List validOptions = Collections.emptyList(); - switch (condition.getType()) { - case CATEGORICAL: - case CATEGORICAL_COMPLEMENT: - validOptions = extractAnalysisFields(); - break; - case NUMERICAL_ACTUAL: - case NUMERICAL_TYPICAL: - case NUMERICAL_DIFF_ABS: - validOptions = getValidFieldNameOptionsForNumeric(rule); - break; - case TIME: - default: - break; - } - if (!validOptions.contains(condition.getFieldName())) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME, validOptions, - condition.getFieldName()); - throw ExceptionsHelper.badRequestException(msg); - } - } + private void validateRule(DetectionRule rule, DetectorFunction function) { + checkFunctionHasRuleSupport(rule, function); + checkScoping(rule); } - private void checkTargetFieldNameIsValid(List analysisFields, String targetFieldName) - throws ElasticsearchParseException { - if (targetFieldName != null && !analysisFields.contains(targetFieldName)) { - String msg = - Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME, analysisFields, targetFieldName); + private void checkFunctionHasRuleSupport(DetectionRule rule, DetectorFunction function) { + if (ruleHasConditionOnResultValue(rule) && FUNCTIONS_WITHOUT_RULE_CONDITION_SUPPORT.contains(function)) { + String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION, function); throw ExceptionsHelper.badRequestException(msg); } } - private List getValidFieldNameOptionsForNumeric(DetectionRule rule) { - List result = new ArrayList<>(); - if (overFieldName != null) { - result.add(byFieldName == null ? overFieldName : byFieldName); - } else if (byFieldName != null) { - result.add(byFieldName); - } - - if (rule.getTargetFieldName() != null) { - ScopingLevel targetLevel = ScopingLevel.from(this, rule.getTargetFieldName()); - result = result.stream().filter(field -> targetLevel.isHigherThan(ScopingLevel.from(this, field))) - .collect(Collectors.toList()); - } - - if (isEmptyFieldNameAllowed(rule)) { - result.add(null); - } - return result; - } - - private boolean isEmptyFieldNameAllowed(DetectionRule rule) { - List analysisFields = extractAnalysisFields(); - return analysisFields.isEmpty() || (rule.getTargetFieldName() != null && analysisFields.size() == 1); - } - - enum ScopingLevel { - PARTITION(3), - OVER(2), - BY(1); - - int level; - - ScopingLevel(int level) { - this.level = level; - } - - boolean isHigherThan(ScopingLevel other) { - return level > other.level; - } - - static ScopingLevel from(Detector.Builder detector, String fieldName) { - if (fieldName.equals(detector.partitionFieldName)) { - return ScopingLevel.PARTITION; - } - if (fieldName.equals(detector.overFieldName)) { - return ScopingLevel.OVER; - } - if (fieldName.equals(detector.byFieldName)) { - return ScopingLevel.BY; + private static boolean ruleHasConditionOnResultValue(DetectionRule rule) { + for (RuleCondition condition : rule.getConditions()) { + switch (condition.getAppliesTo()) { + case ACTUAL: + case TYPICAL: + case DIFF_FROM_TYPICAL: + return true; + case TIME: + return false; + default: + throw new IllegalStateException("Unknown applies_to value [" + condition.getAppliesTo() + "]"); } - throw ExceptionsHelper.badRequestException( - "fieldName '" + fieldName + "' does not match an analysis field"); } + return false; } + private void checkScoping(DetectionRule rule) { + Set analysisFields = new TreeSet<>(extractAnalysisFields()); + rule.getScope().validate(analysisFields); + } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/FilterRef.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/FilterRef.java new file mode 100644 index 0000000000000..7f3fb56287965 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/FilterRef.java @@ -0,0 +1,123 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.job.config; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.MlParserType; + +import java.io.IOException; +import java.util.EnumMap; +import java.util.Locale; +import java.util.Map; +import java.util.Objects; + +public class FilterRef implements ToXContentObject, Writeable { + + public static final ParseField FILTER_REF_FIELD = new ParseField("filter_ref"); + public static final ParseField FILTER_ID = new ParseField("filter_id"); + public static final ParseField FILTER_TYPE = new ParseField("filter_type"); + + public enum FilterType { + INCLUDE, EXCLUDE; + + public static FilterType fromString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); + } + + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); + } + } + + // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly + public static final ConstructingObjectParser METADATA_PARSER = + new ConstructingObjectParser<>(FILTER_REF_FIELD.getPreferredName(), true, + a -> new FilterRef((String) a[0], (FilterType) a[1])); + public static final ConstructingObjectParser CONFIG_PARSER = + new ConstructingObjectParser<>(FILTER_REF_FIELD.getPreferredName(), false, + a -> new FilterRef((String) a[0], (FilterType) a[1])); + public static final Map> PARSERS = new EnumMap<>(MlParserType.class); + + static { + PARSERS.put(MlParserType.METADATA, METADATA_PARSER); + PARSERS.put(MlParserType.CONFIG, CONFIG_PARSER); + for (MlParserType parserType : MlParserType.values()) { + ConstructingObjectParser parser = PARSERS.get(parserType); + assert parser != null; + parser.declareString(ConstructingObjectParser.constructorArg(), FILTER_ID); + parser.declareField(ConstructingObjectParser.optionalConstructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return FilterType.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, FILTER_TYPE, ObjectParser.ValueType.STRING); + } + } + + private final String filterId; + private final FilterType filterType; + + public FilterRef(String filterId, FilterType filterType) { + this.filterId = Objects.requireNonNull(filterId); + this.filterType = filterType == null ? FilterType.INCLUDE : filterType; + } + + public FilterRef(StreamInput in) throws IOException { + filterId = in.readString(); + filterType = in.readEnum(FilterType.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(filterId); + out.writeEnum(filterType); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(FILTER_ID.getPreferredName(), filterId); + builder.field(FILTER_TYPE.getPreferredName(), filterType); + builder.endObject(); + return builder; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof FilterRef == false) { + return false; + } + + FilterRef other = (FilterRef) obj; + return Objects.equals(filterId, other.filterId) && Objects.equals(filterType, other.filterType); + } + + @Override + public int hashCode() { + return Objects.hash(filterId, filterType); + } + + public String getFilterId() { + return filterId; + } + + public FilterType getFilterType() { + return filterType; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 2c7ee538485b9..16243ed16edd4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -496,7 +496,7 @@ public static class DetectorUpdate implements Writeable, ToXContentObject { PARSER.declareInt(ConstructingObjectParser.optionalConstructorArg(), Detector.DETECTOR_INDEX); PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), Job.DESCRIPTION); PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (parser, parseFieldMatcher) -> - DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), Detector.RULES_FIELD); + DetectionRule.CONFIG_PARSER.apply(parser, parseFieldMatcher).build(), Detector.CUSTOM_RULES_FIELD); } private int detectorIndex; @@ -550,7 +550,7 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Job.DESCRIPTION.getPreferredName(), description); } if (rules != null) { - builder.field(Detector.RULES_FIELD.getPreferredName(), rules); + builder.field(Detector.CUSTOM_RULES_FIELD.getPreferredName(), rules); } builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Operator.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Operator.java index 5813a10c93bb3..bfe9b0e3589ba 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Operator.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Operator.java @@ -19,12 +19,6 @@ * Enum representing logical comparisons on doubles */ public enum Operator implements Writeable { - EQ { - @Override - public boolean test(double lhs, double rhs) { - return Double.compare(lhs, rhs) == 0; - } - }, GT { @Override public boolean test(double lhs, double rhs) { @@ -48,19 +42,10 @@ public boolean test(double lhs, double rhs) { public boolean test(double lhs, double rhs) { return Double.compare(lhs, rhs) <= 0; } - }, - MATCH { - @Override - public boolean match(Pattern pattern, String field) { - Matcher match = pattern.matcher(field); - return match.matches(); - } - - @Override - public boolean expectsANumericArgument() { - return false; - } }; + // EQ was considered but given the oddity of such a + // condition and the fact that it would be a numerically + // unstable condition, it was rejected. public static final ParseField OPERATOR_FIELD = new ParseField("operator"); @@ -68,14 +53,6 @@ public boolean test(double lhs, double rhs) { return false; } - public boolean match(Pattern pattern, String field) { - return false; - } - - public boolean expectsANumericArgument() { - return true; - } - public static Operator fromString(String name) { return valueOf(name.trim().toUpperCase(Locale.ROOT)); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java index 607961140be4e..499f3a47f0e06 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleAction.java @@ -13,8 +13,8 @@ import java.util.Locale; public enum RuleAction implements Writeable { - FILTER_RESULTS, - SKIP_SAMPLING; + SKIP_RESULT, + SKIP_MODEL_UPDATE; /** * Case-insensitive from string method. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java index 6ca24c518d8fa..378ceaca6c476 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleCondition.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.core.ml.job.config; -import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -16,29 +15,27 @@ import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.core.ml.MlParserType; -import org.elasticsearch.xpack.core.ml.job.messages.Messages; -import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; import java.io.IOException; import java.util.EnumMap; -import java.util.EnumSet; +import java.util.Locale; import java.util.Map; import java.util.Objects; public class RuleCondition implements ToXContentObject, Writeable { - public static final ParseField TYPE_FIELD = new ParseField("type", "condition_type"); + public static final ParseField RULE_CONDITION_FIELD = new ParseField("rule_condition"); - public static final ParseField FIELD_NAME_FIELD = new ParseField("field_name"); - public static final ParseField FIELD_VALUE_FIELD = new ParseField("field_value"); - public static final ParseField FILTER_ID_FIELD = new ParseField(MlFilter.ID.getPreferredName(), "value_filter"); + + public static final ParseField APPLIES_TO_FIELD = new ParseField("applies_to"); + public static final ParseField VALUE_FIELD = new ParseField("value"); // These parsers follow the pattern that metadata is parsed leniently (to allow for enhancements), whilst config is parsed strictly public static final ConstructingObjectParser METADATA_PARSER = new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), true, - a -> new RuleCondition((RuleConditionType) a[0], (String) a[1], (String) a[2], (Condition) a[3], (String) a[4])); + a -> new RuleCondition((AppliesTo) a[0], (Operator) a[1], (double) a[2])); public static final ConstructingObjectParser CONFIG_PARSER = new ConstructingObjectParser<>(RULE_CONDITION_FIELD.getPreferredName(), false, - a -> new RuleCondition((RuleConditionType) a[0], (String) a[1], (String) a[2], (Condition) a[3], (String) a[4])); + a -> new RuleCondition((AppliesTo) a[0], (Operator) a[1], (double) a[2])); public static final Map> PARSERS = new EnumMap<>(MlParserType.class); @@ -50,111 +47,63 @@ public class RuleCondition implements ToXContentObject, Writeable { assert parser != null; parser.declareField(ConstructingObjectParser.constructorArg(), p -> { if (p.currentToken() == XContentParser.Token.VALUE_STRING) { - return RuleConditionType.fromString(p.text()); + return AppliesTo.fromString(p.text()); + } + throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); + }, APPLIES_TO_FIELD, ValueType.STRING); + parser.declareField(ConstructingObjectParser.constructorArg(), p -> { + if (p.currentToken() == XContentParser.Token.VALUE_STRING) { + return Operator.fromString(p.text()); } throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]"); - }, TYPE_FIELD, ValueType.STRING); - parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_NAME_FIELD); - parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FIELD_VALUE_FIELD); - parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), Condition.PARSER, Condition.CONDITION_FIELD); - parser.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), FILTER_ID_FIELD); + }, Operator.OPERATOR_FIELD, ValueType.STRING); + parser.declareDouble(ConstructingObjectParser.constructorArg(), VALUE_FIELD); } } - private final RuleConditionType type; - private final String fieldName; - private final String fieldValue; - private final Condition condition; - private final String filterId; + private final AppliesTo appliesTo; + private final Operator operator; + private final double value; public RuleCondition(StreamInput in) throws IOException { - type = RuleConditionType.readFromStream(in); - condition = in.readOptionalWriteable(Condition::new); - fieldName = in.readOptionalString(); - fieldValue = in.readOptionalString(); - filterId = in.readOptionalString(); + appliesTo = AppliesTo.readFromStream(in); + operator = Operator.readFromStream(in); + value = in.readDouble(); } @Override public void writeTo(StreamOutput out) throws IOException { - type.writeTo(out); - out.writeOptionalWriteable(condition); - out.writeOptionalString(fieldName); - out.writeOptionalString(fieldValue); - out.writeOptionalString(filterId); + appliesTo.writeTo(out); + operator.writeTo(out); + out.writeDouble(value); } - RuleCondition(RuleConditionType type, String fieldName, String fieldValue, Condition condition, String filterId) { - this.type = type; - this.fieldName = fieldName; - this.fieldValue = fieldValue; - this.condition = condition; - this.filterId = filterId; - - verifyFieldsBoundToType(this); - verifyFieldValueRequiresFieldName(this); - } - - public RuleCondition(RuleCondition ruleCondition) { - this.type = ruleCondition.type; - this.fieldName = ruleCondition.fieldName; - this.fieldValue = ruleCondition.fieldValue; - this.condition = ruleCondition.condition; - this.filterId = ruleCondition.filterId; + public RuleCondition(AppliesTo appliesTo, Operator operator, double value) { + this.appliesTo = appliesTo; + this.operator = operator; + this.value = value; } @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(TYPE_FIELD.getPreferredName(), type); - if (condition != null) { - builder.field(Condition.CONDITION_FIELD.getPreferredName(), condition); - } - if (fieldName != null) { - builder.field(FIELD_NAME_FIELD.getPreferredName(), fieldName); - } - if (fieldValue != null) { - builder.field(FIELD_VALUE_FIELD.getPreferredName(), fieldValue); - } - if (filterId != null) { - builder.field(FILTER_ID_FIELD.getPreferredName(), filterId); - } + builder.field(APPLIES_TO_FIELD.getPreferredName(), appliesTo); + builder.field(Operator.OPERATOR_FIELD.getPreferredName(), operator); + builder.field(VALUE_FIELD.getPreferredName(), value); builder.endObject(); return builder; } - public RuleConditionType getType() { - return type; - } - - /** - * The field name for which the rule applies. Can be null, meaning rule - * applies to all results. - */ - public String getFieldName() { - return fieldName; - } - - /** - * The value of the field name for which the rule applies. When set, the - * rule applies only to the results that have the fieldName/fieldValue pair. - * When null, the rule applies to all values for of the specified field - * name. Only applicable when fieldName is not null. - */ - public String getFieldValue() { - return fieldValue; + public AppliesTo getAppliesTo() { + return appliesTo; } - public Condition getCondition() { - return condition; + public Operator getOperator() { + return operator; } - /** - * The unique identifier of a filter. Required when the rule type is - * categorical. Should be null for all other types. - */ - public String getFilterId() { - return filterId; + public double getValue() { + return value; } @Override @@ -168,114 +117,40 @@ public boolean equals(Object obj) { } RuleCondition other = (RuleCondition) obj; - return Objects.equals(type, other.type) && Objects.equals(fieldName, other.fieldName) - && Objects.equals(fieldValue, other.fieldValue) && Objects.equals(condition, other.condition) - && Objects.equals(filterId, other.filterId); + return appliesTo == other.appliesTo && operator == other.operator && value == other.value; } @Override public int hashCode() { - return Objects.hash(type, fieldName, fieldValue, condition, filterId); - } - - public static RuleCondition createCategorical(String fieldName, String filterId) { - return new RuleCondition(RuleConditionType.CATEGORICAL, fieldName, null, null, filterId); - } - - public static RuleCondition createNumerical(RuleConditionType conditionType, String fieldName, String fieldValue, - Condition condition ) { - if (conditionType.isNumerical() == false) { - throw new IllegalStateException("Rule condition type [" + conditionType + "] not valid for a numerical condition"); - } - return new RuleCondition(conditionType, fieldName, fieldValue, condition, null); + return Objects.hash(appliesTo, operator, value); } public static RuleCondition createTime(Operator operator, long epochSeconds) { - return new RuleCondition(RuleConditionType.TIME, null, null, new Condition(operator, Long.toString(epochSeconds)), null); - } - - private static void verifyFieldsBoundToType(RuleCondition ruleCondition) throws ElasticsearchParseException { - switch (ruleCondition.getType()) { - case CATEGORICAL: - case CATEGORICAL_COMPLEMENT: - verifyCategorical(ruleCondition); - break; - case NUMERICAL_ACTUAL: - case NUMERICAL_TYPICAL: - case NUMERICAL_DIFF_ABS: - verifyNumerical(ruleCondition); - break; - case TIME: - verifyTimeRule(ruleCondition); - break; - default: - throw new IllegalStateException(); - } - } - - private static void verifyCategorical(RuleCondition ruleCondition) throws ElasticsearchParseException { - checkCategoricalHasNoField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); - checkCategoricalHasNoField(RuleCondition.FIELD_VALUE_FIELD.getPreferredName(), ruleCondition.getFieldValue()); - checkCategoricalHasField(FILTER_ID_FIELD.getPreferredName(), ruleCondition.getFilterId()); - } - - private static void checkCategoricalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { - if (fieldValue != null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION, fieldName); - throw ExceptionsHelper.badRequestException(msg); - } - } - - private static void checkCategoricalHasField(String fieldName, Object fieldValue) throws ElasticsearchParseException { - if (fieldValue == null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_MISSING_OPTION, fieldName); - throw ExceptionsHelper.badRequestException(msg); - } + return new RuleCondition(AppliesTo.TIME, operator, epochSeconds); } - private static void verifyNumerical(RuleCondition ruleCondition) throws ElasticsearchParseException { - checkNumericalHasNoField(FILTER_ID_FIELD.getPreferredName(), ruleCondition.getFilterId()); - checkNumericalHasField(Condition.CONDITION_FIELD.getPreferredName(), ruleCondition.getCondition()); - if (ruleCondition.getFieldName() != null && ruleCondition.getFieldValue() == null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE); - throw ExceptionsHelper.badRequestException(msg); - } - checkNumericalConditionOparatorsAreValid(ruleCondition); - } + public enum AppliesTo implements Writeable { + ACTUAL, + TYPICAL, + DIFF_FROM_TYPICAL, + TIME; - private static void checkNumericalHasNoField(String fieldName, Object fieldValue) throws ElasticsearchParseException { - if (fieldValue != null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPTION, fieldName); - throw ExceptionsHelper.badRequestException(msg); + public static AppliesTo fromString(String value) { + return valueOf(value.toUpperCase(Locale.ROOT)); } - } - private static void checkNumericalHasField(String fieldName, Object fieldValue) throws ElasticsearchParseException { - if (fieldValue == null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_MISSING_OPTION, fieldName); - throw ExceptionsHelper.badRequestException(msg); + public static AppliesTo readFromStream(StreamInput in) throws IOException { + return in.readEnum(AppliesTo.class); } - } - private static void verifyFieldValueRequiresFieldName(RuleCondition ruleCondition) throws ElasticsearchParseException { - if (ruleCondition.getFieldValue() != null && ruleCondition.getFieldName() == null) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_MISSING_FIELD_NAME, - ruleCondition.getFieldValue()); - throw ExceptionsHelper.badRequestException(msg); + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(this); } - } - - static EnumSet VALID_CONDITION_OPERATORS = EnumSet.of(Operator.LT, Operator.LTE, Operator.GT, Operator.GTE); - private static void checkNumericalConditionOparatorsAreValid(RuleCondition ruleCondition) throws ElasticsearchParseException { - Operator operator = ruleCondition.getCondition().getOperator(); - if (!VALID_CONDITION_OPERATORS.contains(operator)) { - String msg = Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPERATOR, operator); - throw ExceptionsHelper.badRequestException(msg); + @Override + public String toString() { + return name().toLowerCase(Locale.ROOT); } } - - private static void verifyTimeRule(RuleCondition ruleCondition) { - checkNumericalConditionOparatorsAreValid(ruleCondition); - } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionType.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionType.java deleted file mode 100644 index aa563d001b5ca..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionType.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.core.ml.job.config; - - -import org.elasticsearch.Version; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; - -import java.io.IOException; -import java.util.Locale; - -public enum RuleConditionType implements Writeable { - CATEGORICAL(false, true), - NUMERICAL_ACTUAL(true, false), - NUMERICAL_TYPICAL(true, false), - NUMERICAL_DIFF_ABS(true, false), - TIME(false, false), - CATEGORICAL_COMPLEMENT(false, true); - - private final boolean isNumerical; - private final boolean isCategorical; - - RuleConditionType(boolean isNumerical, boolean isCategorical) { - this.isNumerical = isNumerical; - this.isCategorical = isCategorical; - } - - public boolean isNumerical() { - return isNumerical; - } - - public boolean isCategorical() { - return isCategorical; - } - - /** - * Case-insensitive from string method. - * - * @param value - * String representation - * @return The condition type - */ - public static RuleConditionType fromString(String value) { - return RuleConditionType.valueOf(value.toUpperCase(Locale.ROOT)); - } - - public static RuleConditionType readFromStream(StreamInput in) throws IOException { - return in.readEnum(RuleConditionType.class); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - if (this == CATEGORICAL_COMPLEMENT && out.getVersion().before(Version.V_6_3_0)) { - out.writeEnum(CATEGORICAL); - } else { - out.writeEnum(this); - } - } - - @Override - public String toString() { - return name().toLowerCase(Locale.ROOT); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java new file mode 100644 index 0000000000000..b6b3b4e061bdd --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.job.config; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ContextParser; +import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.xpack.core.ml.MlParserType; +import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class RuleScope implements ToXContentObject, Writeable { + + public static ContextParser parser(MlParserType parserType) { + return (p, c) -> { + Map unparsedScope = p.map(); + if (unparsedScope.isEmpty()) { + return new RuleScope(); + } + ConstructingObjectParser filterRefParser = FilterRef.PARSERS.get(parserType); + Map scope = new HashMap<>(); + for (Map.Entry entry : unparsedScope.entrySet()) { + try (XContentBuilder builder = XContentFactory.jsonBuilder()) { + builder.map((Map) entry.getValue()); + try (XContentParser scopeParser = XContentFactory.xContent(builder.contentType()).createParser( + NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, Strings.toString(builder))) { + scope.put(entry.getKey(), filterRefParser.parse(scopeParser, null)); + } + } + } + return new RuleScope(scope); + }; + } + + private final Map scope; + + public RuleScope() { + scope = Collections.emptyMap(); + } + + public RuleScope(Map scope) { + this.scope = Objects.requireNonNull(scope); + } + + public RuleScope(StreamInput in) throws IOException { + scope = in.readMap(StreamInput::readString, FilterRef::new); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeMap(scope, StreamOutput::writeString, (out1, value) -> value.writeTo(out1)); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.map(scope); + } + + public boolean isEmpty() { + return scope.isEmpty(); + } + + public void validate(Set validKeys) { + Optional invalidKey = scope.keySet().stream().filter(k -> !validKeys.contains(k)).findFirst(); + if (invalidKey.isPresent()) { + throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_SCOPE_HAS_INVALID_FIELD, + invalidKey.get(), validKeys)); + } + } + + public Set getReferencedFilters() { + return scope.values().stream().map(FilterRef::getFilterId).collect(Collectors.toSet()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + + if (obj instanceof RuleScope == false) { + return false; + } + + RuleScope other = (RuleScope) obj; + return Objects.equals(scope, other.scope); + } + + @Override + public int hashCode() { + return Objects.hash(scope); + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + + private Map scope = new HashMap<>(); + + public Builder() {} + + public Builder(RuleScope otherScope) { + scope = new HashMap<>(otherScope.scope); + } + + public Builder exclude(String field, String filterId) { + scope.put(field, new FilterRef(filterId, FilterRef.FilterType.EXCLUDE)); + return this; + } + + public Builder include(String field, String filterId) { + scope.put(field, new FilterRef(filterId, FilterRef.FilterType.INCLUDE)); + return this; + } + + public RuleScope build() { + return new RuleScope(scope); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java index 7e5dc231e057a..79d8f068d91f8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/messages/Messages.java @@ -88,35 +88,12 @@ public final class Messages { "categorization_filters require setting categorization_field_name"; public static final String JOB_CONFIG_CATEGORIZATION_ANALYZER_REQUIRES_CATEGORIZATION_FIELD_NAME = "categorization_analyzer requires setting categorization_field_name"; - public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_NULL = "Invalid condition: the value field cannot be null"; - public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER = - "Invalid condition value: cannot parse a double from string ''{0}''"; - public static final String JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX = - "Invalid condition value: ''{0}'' is not a valid regular expression"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_INVALID_OPTION = - "Invalid detector rule: a categorical rule_condition does not support {0}"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_CATEGORICAL_MISSING_OPTION = - "Invalid detector rule: a categorical rule_condition requires {0} to be set"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME = - "Invalid detector rule: field_name has to be one of {0}; actual was ''{1}''"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_MISSING_FIELD_NAME = - "Invalid detector rule: missing field_name in rule_condition where field_value ''{0}'' is set"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPERATOR = - "Invalid detector rule: operator ''{0}'' is not allowed"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_INVALID_OPTION = - "Invalid detector rule: a numerical rule_condition does not support {0}"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_MISSING_OPTION = - "Invalid detector rule: a numerical rule_condition requires {0} to be set"; - public static final String JOB_CONFIG_DETECTION_RULE_CONDITION_NUMERICAL_WITH_FIELD_NAME_REQUIRES_FIELD_VALUE = - "Invalid detector rule: a numerical rule_condition with field_name requires that field_value is set"; - public static final String JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME = - "Invalid detector rule: target_field_name has to be one of {0}; actual was ''{1}''"; - public static final String JOB_CONFIG_DETECTION_RULE_MISSING_TARGET_FIELD_NAME = - "Invalid detector rule: missing target_field_name where target_field_value ''{0}'' is set"; public static final String JOB_CONFIG_DETECTION_RULE_NOT_SUPPORTED_BY_FUNCTION = - "Invalid detector rule: function {0} does not support rules"; - public static final String JOB_CONFIG_DETECTION_RULE_REQUIRES_AT_LEAST_ONE_CONDITION = - "Invalid detector rule: at least one rule_condition is required"; + "Invalid detector rule: function {0} does not support rules with conditions"; + public static final String JOB_CONFIG_DETECTION_RULE_REQUIRES_SCOPE_OR_CONDITION = + "Invalid detector rule: at least scope or a condition is required"; + public static final String JOB_CONFIG_DETECTION_RULE_SCOPE_HAS_INVALID_FIELD = + "Invalid detector rule: scope field ''{0}'' is invalid; select from {1}"; public static final String JOB_CONFIG_FIELDNAME_INCOMPATIBLE_FUNCTION = "field_name cannot be used with function ''{0}''"; public static final String JOB_CONFIG_FIELD_VALUE_TOO_LOW = "{0} cannot be less than {1,number}. Value = {2,number}"; public static final String JOB_CONFIG_MODEL_MEMORY_LIMIT_TOO_LOW = "model_memory_limit must be at least 1 MiB. Value = {0,number}"; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java index f26c67935edff..67b83646c4237 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/MetricConfig.java @@ -12,7 +12,7 @@ import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.ToXContentFragment; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.mapper.NumberFieldMapper; @@ -75,12 +75,11 @@ public class MetricConfig implements Writeable, ToXContentFragment { MAPPER_TYPES = types; } - public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - NAME, a -> new MetricConfig((String)a[0], (List) a[1])); + public static final ObjectParser PARSER = new ObjectParser<>(NAME, MetricConfig.Builder::new); static { - PARSER.declareString(ConstructingObjectParser.constructorArg(), FIELD); - PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), METRICS); + PARSER.declareString(MetricConfig.Builder::setField, FIELD); + PARSER.declareStringArray(MetricConfig.Builder::setMetrics, METRICS); } MetricConfig(String name, List metrics) { @@ -257,4 +256,4 @@ public MetricConfig build() { return new MetricConfig(field, metrics); } } -} \ No newline at end of file +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java index 3818ebcf44758..422ecdd5fd9fb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rollup/job/RollupJobConfig.java @@ -63,7 +63,7 @@ public class RollupJobConfig implements NamedWriteable, ToXContentObject { static { PARSER.declareString(RollupJobConfig.Builder::setId, RollupField.ID); PARSER.declareObject(RollupJobConfig.Builder::setGroupConfig, (p, c) -> GroupConfig.PARSER.apply(p,c).build(), GROUPS); - PARSER.declareObjectArray(RollupJobConfig.Builder::setMetricsConfig, MetricConfig.PARSER, METRICS); + PARSER.declareObjectArray(RollupJobConfig.Builder::setMetricsConfig, (p, c) -> MetricConfig.PARSER.apply(p, c).build(), METRICS); PARSER.declareString((params, val) -> params.setTimeout(TimeValue.parseTimeValue(val, TIMEOUT.getPreferredName())), TIMEOUT); PARSER.declareString(RollupJobConfig.Builder::setIndexPattern, INDEX_PATTERN); diff --git a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy index 1f1bd66005693..0cd7a32bcc47b 100644 --- a/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/core/src/main/plugin-metadata/plugin-security.policy @@ -15,8 +15,6 @@ grant { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - - permission java.lang.RuntimePermission "setContextClassLoader"; }; grant codeBase "${codebase.netty-transport}" { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java index 57ba7a0c4674d..e9c9ba95bfd38 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/AbstractLicensesIntegrationTestCase.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.license; import org.elasticsearch.analysis.common.CommonAnalysisPlugin; @@ -14,16 +13,16 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.XPackSettings; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; import java.util.Arrays; import java.util.Collection; import java.util.concurrent.CountDownLatch; -public abstract class AbstractLicensesIntegrationTestCase extends XPackIntegTestCase { +public abstract class AbstractLicensesIntegrationTestCase extends ESIntegTestCase { @Override protected Settings nodeSettings(int nodeOrdinal) { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java index 2b0d60f321c92..fc8e25e3ccca9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/StartBasicLicenseTests.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.license; import org.elasticsearch.client.Response; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java index f98eb9d5dcecc..3d6c317542279 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/calendars/ScheduledEventTests.java @@ -11,12 +11,10 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.ml.job.config.Connective; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Operator; import org.elasticsearch.xpack.core.ml.job.config.RuleAction; import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; -import org.elasticsearch.xpack.core.ml.job.config.RuleConditionType; import org.joda.time.DateTime; import java.io.IOException; @@ -56,25 +54,22 @@ public void testToDetectionRule() { ScheduledEvent event = createTestInstance(); DetectionRule rule = event.toDetectionRule(TimeValue.timeValueSeconds(bucketSpanSecs)); - assertEquals(Connective.AND, rule.getConditionsConnective()); - assertEquals(rule.getActions(), EnumSet.of(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING)); - assertNull(rule.getTargetFieldName()); - assertNull(rule.getTargetFieldValue()); + assertEquals(rule.getActions(), EnumSet.of(RuleAction.SKIP_RESULT, RuleAction.SKIP_MODEL_UPDATE)); List conditions = rule.getConditions(); assertEquals(2, conditions.size()); - assertEquals(RuleConditionType.TIME, conditions.get(0).getType()); - assertEquals(RuleConditionType.TIME, conditions.get(1).getType()); - assertEquals(Operator.GTE, conditions.get(0).getCondition().getOperator()); - assertEquals(Operator.LT, conditions.get(1).getCondition().getOperator()); + assertEquals(RuleCondition.AppliesTo.TIME, conditions.get(0).getAppliesTo()); + assertEquals(RuleCondition.AppliesTo.TIME, conditions.get(1).getAppliesTo()); + assertEquals(Operator.GTE, conditions.get(0).getOperator()); + assertEquals(Operator.LT, conditions.get(1).getOperator()); // Check times are aligned with the bucket - long conditionStartTime = Long.parseLong(conditions.get(0).getCondition().getValue()); + long conditionStartTime = (long) conditions.get(0).getValue(); assertEquals(0, conditionStartTime % bucketSpanSecs); long bucketCount = conditionStartTime / bucketSpanSecs; assertEquals(bucketSpanSecs * bucketCount, conditionStartTime); - long conditionEndTime = Long.parseLong(conditions.get(1).getCondition().getValue()); + long conditionEndTime = (long) conditions.get(1).getValue(); assertEquals(0, conditionEndTime % bucketSpanSecs); long eventTime = event.getEndTime().toEpochSecond() - conditionStartTime; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java index 6aa987fc0e932..d59ef16dfdf2c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfigTests.java @@ -40,7 +40,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.TimeZone; @@ -193,11 +192,11 @@ public void testDefaults() { public void testDefaultQueryDelay() { DatafeedConfig.Builder feedBuilder1 = new DatafeedConfig.Builder("datafeed1", "job1"); - feedBuilder1.setIndices(Arrays.asList("foo")); + feedBuilder1.setIndices(Collections.singletonList("foo")); DatafeedConfig.Builder feedBuilder2 = new DatafeedConfig.Builder("datafeed2", "job1"); - feedBuilder2.setIndices(Arrays.asList("foo")); + feedBuilder2.setIndices(Collections.singletonList("foo")); DatafeedConfig.Builder feedBuilder3 = new DatafeedConfig.Builder("datafeed3", "job2"); - feedBuilder3.setIndices(Arrays.asList("foo")); + feedBuilder3.setIndices(Collections.singletonList("foo")); DatafeedConfig feed1 = feedBuilder1.build(); DatafeedConfig feed2 = feedBuilder2.build(); DatafeedConfig feed3 = feedBuilder3.build(); @@ -208,19 +207,19 @@ public void testDefaultQueryDelay() { assertThat(feed1.getQueryDelay(), not(equalTo(feed3.getQueryDelay()))); } - public void testCheckValid_GivenNullIndices() throws IOException { + public void testCheckValid_GivenNullIndices() { DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); expectThrows(IllegalArgumentException.class, () -> conf.setIndices(null)); } - public void testCheckValid_GivenEmptyIndices() throws IOException { + public void testCheckValid_GivenEmptyIndices() { DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); conf.setIndices(Collections.emptyList()); ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, conf::build); assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indices", "[]"), e.getMessage()); } - public void testCheckValid_GivenIndicesContainsOnlyNulls() throws IOException { + public void testCheckValid_GivenIndicesContainsOnlyNulls() { List indices = new ArrayList<>(); indices.add(null); indices.add(null); @@ -230,7 +229,7 @@ public void testCheckValid_GivenIndicesContainsOnlyNulls() throws IOException { assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indices", "[null, null]"), e.getMessage()); } - public void testCheckValid_GivenIndicesContainsOnlyEmptyStrings() throws IOException { + public void testCheckValid_GivenIndicesContainsOnlyEmptyStrings() { List indices = new ArrayList<>(); indices.add(""); indices.add(""); @@ -240,27 +239,27 @@ public void testCheckValid_GivenIndicesContainsOnlyEmptyStrings() throws IOExcep assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "indices", "[, ]"), e.getMessage()); } - public void testCheckValid_GivenNegativeQueryDelay() throws IOException { + public void testCheckValid_GivenNegativeQueryDelay() { DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setQueryDelay(TimeValue.timeValueMillis(-10))); assertEquals("query_delay cannot be less than 0. Value = -10", e.getMessage()); } - public void testCheckValid_GivenZeroFrequency() throws IOException { + public void testCheckValid_GivenZeroFrequency() { DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setFrequency(TimeValue.ZERO)); assertEquals("frequency cannot be less or equal than 0. Value = 0s", e.getMessage()); } - public void testCheckValid_GivenNegativeFrequency() throws IOException { + public void testCheckValid_GivenNegativeFrequency() { DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); IllegalArgumentException e = ESTestCase.expectThrows(IllegalArgumentException.class, () -> conf.setFrequency(TimeValue.timeValueMinutes(-1))); assertEquals("frequency cannot be less or equal than 0. Value = -1", e.getMessage()); } - public void testCheckValid_GivenNegativeScrollSize() throws IOException { + public void testCheckValid_GivenNegativeScrollSize() { DatafeedConfig.Builder conf = new DatafeedConfig.Builder("datafeed1", "job1"); ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, () -> conf.setScrollSize(-1000)); assertEquals(Messages.getMessage(Messages.DATAFEED_CONFIG_INVALID_OPTION_VALUE, "scroll_size", -1000L), e.getMessage()); @@ -414,7 +413,7 @@ public void testDefaultFrequency_GivenNegative() { public void testDefaultFrequency_GivenNoAggregations() { DatafeedConfig.Builder datafeedBuilder = new DatafeedConfig.Builder("feed", "job"); - datafeedBuilder.setIndices(Arrays.asList("my_index")); + datafeedBuilder.setIndices(Collections.singletonList("my_index")); DatafeedConfig datafeed = datafeedBuilder.build(); assertEquals(TimeValue.timeValueMinutes(1), datafeed.defaultFrequency(TimeValue.timeValueSeconds(1))); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java index e64c01d7d0c3b..6c54eb78189a4 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfigTests.java @@ -486,11 +486,9 @@ public void testEquals_GivenDifferentCategorizationFilters() { assertFalse(config2.equals(config1)); } - public void testExtractReferencedLists() { - DetectionRule rule1 = new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("foo", - "filter1"))).build(); - DetectionRule rule2 = new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("foo", - "filter2"))).build(); + public void testExtractReferencedFilters() { + DetectionRule rule1 = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "filter1")).build(); + DetectionRule rule2 = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "filter2")).build(); Detector.Builder detector1 = new Detector.Builder("count", null); detector1.setByFieldName("foo"); detector1.setRules(Collections.singletonList(rule1)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java index 3aaf99ab730f8..a57bd24eda0e1 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectionRuleTests.java @@ -5,100 +5,40 @@ */ package org.elasticsearch.xpack.core.ml.job.config; +import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; -import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; -public class DetectionRuleTests extends AbstractSerializingTestCase { - - public void testExtractReferencedLists() { - RuleCondition numericalCondition = - new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "field", "value", new Condition(Operator.GT, "5"), null); - List conditions = Arrays.asList( - numericalCondition, - RuleCondition.createCategorical("foo", "filter1"), - RuleCondition.createCategorical("bar", "filter2")); - - DetectionRule rule = new DetectionRule.Builder(conditions).build(); - - assertEquals(new HashSet<>(Arrays.asList("filter1", "filter2")), rule.extractReferencedFilters()); - } - - public void testEqualsGivenSameObject() { - DetectionRule rule = createFullyPopulated().build(); - assertTrue(rule.equals(rule)); - } - - public void testEqualsGivenString() { - assertFalse(createFullyPopulated().build().equals("a string")); - } +import static org.hamcrest.Matchers.equalTo; - public void testEqualsGivenDifferentTargetFieldName() { - DetectionRule rule1 = createFullyPopulated().build(); - DetectionRule rule2 = createFullyPopulated().setTargetFieldName("targetField2").build(); - assertFalse(rule1.equals(rule2)); - assertFalse(rule2.equals(rule1)); - } - - public void testEqualsGivenDifferentTargetFieldValue() { - DetectionRule rule1 = createFullyPopulated().build(); - DetectionRule rule2 = createFullyPopulated().setTargetFieldValue("targetValue2").build(); - assertFalse(rule1.equals(rule2)); - assertFalse(rule2.equals(rule1)); - } - - public void testEqualsGivenDifferentConnective() { - DetectionRule rule1 = createFullyPopulated().build(); - DetectionRule rule2 = createFullyPopulated().setConditionsConnective(Connective.OR).build(); - assertFalse(rule1.equals(rule2)); - assertFalse(rule2.equals(rule1)); - } - - public void testEqualsGivenRules() { - DetectionRule rule1 = createFullyPopulated().build(); - DetectionRule rule2 = createFullyPopulated().setConditions(createRule("10")).build(); - assertFalse(rule1.equals(rule2)); - assertFalse(rule2.equals(rule1)); - } +public class DetectionRuleTests extends AbstractSerializingTestCase { - public void testEqualsGivenEqual() { - DetectionRule rule1 = createFullyPopulated().build(); - DetectionRule rule2 = createFullyPopulated().build(); - assertTrue(rule1.equals(rule2)); - assertTrue(rule2.equals(rule1)); - assertEquals(rule1.hashCode(), rule2.hashCode()); + public void testBuildWithNeitherScopeNorCondition() { + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, () -> new DetectionRule.Builder().build()); + assertThat(e.getMessage(), equalTo("Invalid detector rule: at least scope or a condition is required")); } - private static DetectionRule.Builder createFullyPopulated() { - return new DetectionRule.Builder(createRule("5")) - .setActions(EnumSet.of(RuleAction.FILTER_RESULTS, RuleAction.SKIP_SAMPLING)) - .setTargetFieldName("targetField") - .setTargetFieldValue("targetValue") - .setConditionsConnective(Connective.AND); - } + public void testExtractReferencedLists() { + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder() + .exclude("foo", "filter1").include("bar", "filter2")) + .build(); - private static List createRule(String value) { - Condition condition = new Condition(Operator.GT, value); - return Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); + assertEquals(new HashSet<>(Arrays.asList("filter1", "filter2")), rule.extractReferencedFilters()); } @Override protected DetectionRule createTestInstance() { - int size = 1 + randomInt(20); - List ruleConditions = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - // no need for random condition (it is already tested) - ruleConditions.addAll(createRule(Double.toString(randomDouble()))); - } - DetectionRule.Builder builder = new DetectionRule.Builder(ruleConditions); + DetectionRule.Builder builder = new DetectionRule.Builder(); if (randomBoolean()) { EnumSet actions = EnumSet.noneOf(RuleAction.class); @@ -109,13 +49,35 @@ protected DetectionRule createTestInstance() { builder.setActions(actions); } - if (randomBoolean()) { - builder.setConditionsConnective(randomFrom(Connective.values())); + boolean hasScope = randomBoolean(); + boolean hasConditions = randomBoolean(); + + if (!hasScope && !hasConditions) { + // at least one of the two should be present + if (randomBoolean()) { + hasScope = true; + } else { + hasConditions = true; + } } - if (randomBoolean()) { - builder.setTargetFieldName(randomAlphaOfLengthBetween(1, 20)); - builder.setTargetFieldValue(randomAlphaOfLengthBetween(1, 20)); + if (hasScope) { + Map scope = new HashMap<>(); + int scopeSize = randomIntBetween(1, 3); + for (int i = 0; i < scopeSize; i++) { + scope.put(randomAlphaOfLength(20), new FilterRef(randomAlphaOfLength(20), randomFrom(FilterRef.FilterType.values()))); + } + builder.setScope(new RuleScope(scope)); + } + + if (hasConditions) { + int size = randomIntBetween(1, 5); + List ruleConditions = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + // no need for random condition (it is already tested) + ruleConditions.addAll(createCondition(randomDouble())); + } + builder.setConditions(ruleConditions); } return builder.build(); @@ -132,39 +94,34 @@ protected DetectionRule doParseInstance(XContentParser parser) { } @Override - protected DetectionRule mutateInstance(DetectionRule instance) throws IOException { + protected DetectionRule mutateInstance(DetectionRule instance) { List conditions = instance.getConditions(); + RuleScope scope = instance.getScope(); EnumSet actions = instance.getActions(); - String targetFieldName = instance.getTargetFieldName(); - String targetFieldValue = instance.getTargetFieldValue(); - Connective connective = instance.getConditionsConnective(); - switch (between(0, 3)) { + switch (between(0, 2)) { case 0: - conditions = new ArrayList<>(conditions); - conditions.addAll(createRule(Double.toString(randomDouble()))); + if (actions.size() == RuleAction.values().length) { + actions = EnumSet.of(randomFrom(RuleAction.values())); + } else { + actions = EnumSet.allOf(RuleAction.class); + } break; case 1: - targetFieldName = randomAlphaOfLengthBetween(5, 10); + conditions = new ArrayList<>(conditions); + conditions.addAll(createCondition(randomDouble())); break; case 2: - targetFieldValue = randomAlphaOfLengthBetween(5, 10); - if (targetFieldName == null) { - targetFieldName = randomAlphaOfLengthBetween(5, 10); - } - break; - case 3: - if (connective == Connective.AND) { - connective = Connective.OR; - } else { - connective = Connective.AND; - } + scope = new RuleScope.Builder(scope).include("another_field", "another_filter").build(); break; default: throw new AssertionError("Illegal randomisation branch"); } - return new DetectionRule.Builder(conditions).setActions(actions).setTargetFieldName(targetFieldName) - .setTargetFieldValue(targetFieldValue).setConditionsConnective(connective).build(); + return new DetectionRule.Builder(conditions).setActions(actions).setScope(scope).build(); + } + + private static List createCondition(double value) { + return Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, value)); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java index 1296928d68478..c9be0d3b89248 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DetectorTests.java @@ -63,55 +63,46 @@ public void testEquals_GivenDifferentByFieldName() { Detector.Builder builder = new Detector.Builder(detector2); builder.setByFieldName("by2"); - Condition condition = new Condition(Operator.GT, "5"); - DetectionRule rule = new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by2", "val", condition, null))) - .setActions(RuleAction.FILTER_RESULTS).setTargetFieldName("over_field") - .setTargetFieldValue("targetValue") - .setConditionsConnective(Connective.AND) - .build(); - builder.setRules(Collections.singletonList(rule)); detector2 = builder.build(); assertFalse(detector1.equals(detector2)); } public void testExtractAnalysisFields() { - Detector detector = createDetector().build(); + DetectionRule rule = new DetectionRule.Builder( + Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5))) + .setActions(RuleAction.SKIP_RESULT) + .build(); + Detector.Builder builder = createDetector(); + builder.setRules(Collections.singletonList(rule)); + Detector detector = builder.build(); assertEquals(Arrays.asList("by_field", "over_field", "partition"), detector.extractAnalysisFields()); - Detector.Builder builder = new Detector.Builder(detector); + builder.setPartitionFieldName(null); + detector = builder.build(); + assertEquals(Arrays.asList("by_field", "over_field"), detector.extractAnalysisFields()); + builder = new Detector.Builder(detector); - Condition condition = new Condition(Operator.GT, "5"); - DetectionRule rule = new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setActions(RuleAction.FILTER_RESULTS) - .setTargetFieldName("over_field") - .setTargetFieldValue("targetValue") - .setConditionsConnective(Connective.AND) - .build(); - builder.setRules(Collections.singletonList(rule)); builder.setByFieldName(null); + detector = builder.build(); + assertEquals(Collections.singletonList("over_field"), detector.extractAnalysisFields()); + builder = new Detector.Builder(detector); - rule = new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setActions(RuleAction.FILTER_RESULTS) - .setConditionsConnective(Connective.AND) - .build(); - builder.setRules(Collections.singletonList(rule)); builder.setOverFieldName(null); + detector = builder.build(); + assertTrue(detector.extractAnalysisFields().isEmpty()); } public void testExtractReferencedLists() { Detector.Builder builder = createDetector(); builder.setRules(Arrays.asList( - new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list1"))).build(), - new DetectionRule.Builder(Collections.singletonList(RuleCondition.createCategorical("by_field", "list2"))).build())); + new DetectionRule.Builder(RuleScope.builder().exclude("by_field", "list1")).build(), + new DetectionRule.Builder(RuleScope.builder().exclude("by_field", "list2")).build())); Detector detector = builder.build(); assertEquals(new HashSet<>(Arrays.asList("list1", "list2")), detector.extractReferencedFilters()); @@ -139,13 +130,8 @@ private Detector.Builder createDetector() { detector.setOverFieldName("over_field"); detector.setPartitionFieldName("partition"); detector.setUseNull(true); - Condition condition = new Condition(Operator.GT, "5"); - DetectionRule rule = new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "by_field", "val", condition, null))) - .setActions(RuleAction.FILTER_RESULTS) - .setTargetFieldName("over_field") - .setTargetFieldValue("targetValue") - .setConditionsConnective(Connective.AND) + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder().exclude("partition", "partition_filter")) + .setActions(RuleAction.SKIP_RESULT) .build(); detector.setRules(Collections.singletonList(rule)); return detector; @@ -159,32 +145,27 @@ protected Detector createTestInstance() { detector = new Detector.Builder(function = randomFrom(Detector.COUNT_WITHOUT_FIELD_FUNCTIONS), null); } else { EnumSet functions = EnumSet.copyOf(Detector.FIELD_NAME_FUNCTIONS); - functions.removeAll(Detector.Builder.FUNCTIONS_WITHOUT_RULE_SUPPORT); detector = new Detector.Builder(function = randomFrom(functions), randomAlphaOfLengthBetween(1, 20)); } if (randomBoolean()) { detector.setDetectorDescription(randomAlphaOfLengthBetween(1, 20)); } - String fieldName = null; if (randomBoolean()) { - detector.setPartitionFieldName(fieldName = randomAlphaOfLengthBetween(6, 20)); + detector.setPartitionFieldName(randomAlphaOfLengthBetween(6, 20)); } else if (randomBoolean() && Detector.NO_OVER_FIELD_NAME_FUNCTIONS.contains(function) == false) { - detector.setOverFieldName(fieldName = randomAlphaOfLengthBetween(6, 20)); + detector.setOverFieldName(randomAlphaOfLengthBetween(6, 20)); } else if (randomBoolean()) { - detector.setByFieldName(fieldName = randomAlphaOfLengthBetween(6, 20)); + detector.setByFieldName(randomAlphaOfLengthBetween(6, 20)); } if (randomBoolean()) { detector.setExcludeFrequent(randomFrom(Detector.ExcludeFrequent.values())); } - if (randomBoolean()) { + if (Detector.FUNCTIONS_WITHOUT_RULE_CONDITION_SUPPORT.contains(function) == false && randomBoolean()) { int size = randomInt(10); List rules = new ArrayList<>(size); for (int i = 0; i < size; i++) { // no need for random DetectionRule (it is already tested) - Condition condition = new Condition(Operator.GT, "5"); - rules.add(new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setTargetFieldName(fieldName).build()); + rules.add(new DetectionRule.Builder(Collections.singletonList(RuleConditionTests.createRandom())).build()); } detector.setRules(rules); } @@ -462,63 +443,20 @@ public void testVerify_GivenFunctionsThatCanHaveByField() { } } - public void testVerify_GivenInvalidRuleTargetFieldName() { - Detector.Builder detector = new Detector.Builder("mean", "metricVale"); - detector.setByFieldName("metricName"); - detector.setPartitionFieldName("instance"); - RuleCondition ruleCondition = - new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricVale", new Condition(Operator.LT, "5"), null); - DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instancE").build(); - detector.setRules(Collections.singletonList(rule)); - - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); - - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_INVALID_TARGET_FIELD_NAME, - "[metricName, instance]", "instancE"), - e.getMessage()); - } - - public void testVerify_GivenValidRule() { - Detector.Builder detector = new Detector.Builder("mean", "metricVale"); - detector.setByFieldName("metricName"); - detector.setPartitionFieldName("instance"); - RuleCondition ruleCondition = - new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "5"), null); - DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instance").build(); - detector.setRules(Collections.singletonList(rule)); - detector.build(); - } - - public void testVerify_GivenCategoricalRuleOnAllPartitioningFields() { + public void testVerify_GivenAllPartitioningFieldsAreScoped() { Detector.Builder detector = new Detector.Builder("count", null); detector.setPartitionFieldName("my_partition"); detector.setOverFieldName("my_over"); detector.setByFieldName("my_by"); - DetectionRule rule = new DetectionRule.Builder(Arrays.asList( - RuleCondition.createCategorical("my_partition", "my_filter_id"), - RuleCondition.createCategorical("my_over", "my_filter_id"), - RuleCondition.createCategorical("my_by", "my_filter_id") - )).build(); - detector.setRules(Collections.singletonList(rule)); - detector.build(); - } - - public void testVerify_GivenCategoricalRuleOnInvalidField() { - Detector.Builder detector = new Detector.Builder("mean", "my_metric"); - detector.setPartitionFieldName("my_partition"); - detector.setOverFieldName("my_over"); - detector.setByFieldName("my_by"); - DetectionRule rule = new DetectionRule.Builder(Collections.singletonList( - RuleCondition.createCategorical("my_metric", "my_filter_id") - )).build(); + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder() + .exclude("my_partition", "my_filter_id") + .exclude("my_over", "my_filter_id") + .exclude("my_by", "my_filter_id")) + .build(); detector.setRules(Collections.singletonList(rule)); - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); - - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_CONDITION_INVALID_FIELD_NAME, - "[my_by, my_over, my_partition]", "my_metric"), - e.getMessage()); + detector.build(); } public void testVerify_GivenSameByAndPartition() { @@ -596,6 +534,85 @@ public void testVerify_GivenOverIsOver() { assertEquals("'over' is not a permitted value for over_field_name", e.getMessage()); } + public void testVerify_GivenRulesAndFunctionIsLatLong() { + Detector.Builder detector = new Detector.Builder("lat_long", "geo"); + detector.setRules(Collections.singletonList(new DetectionRule.Builder(Collections.singletonList( + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 42.0))).build())); + + ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); + + assertThat(e.getMessage(), equalTo("Invalid detector rule: function lat_long does not support rules with conditions")); + } + + public void testVerify_GivenRulesAndFunctionIsMetric() { + Detector.Builder detector = new Detector.Builder("metric", "some_metric"); + detector.setRules(Collections.singletonList(new DetectionRule.Builder(Collections.singletonList( + new RuleCondition(RuleCondition.AppliesTo.TYPICAL, Operator.GT, 42.0))).build())); + + ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); + + assertThat(e.getMessage(), equalTo("Invalid detector rule: function metric does not support rules with conditions")); + } + + public void testVerify_GivenRulesAndFunctionIsRare() { + Detector.Builder detector = new Detector.Builder("rare", null); + detector.setByFieldName("some_field"); + detector.setRules(Collections.singletonList(new DetectionRule.Builder(Collections.singletonList( + new RuleCondition(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, Operator.GT, 42.0))).build())); + + ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); + + assertThat(e.getMessage(), equalTo("Invalid detector rule: function rare does not support rules with conditions")); + } + + public void testVerify_GivenRulesAndFunctionIsFreqRare() { + Detector.Builder detector = new Detector.Builder("freq_rare", null); + detector.setByFieldName("some_field"); + detector.setOverFieldName("some_field2"); + detector.setRules(Collections.singletonList(new DetectionRule.Builder(Collections.singletonList( + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 42.0))).build())); + + ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); + + assertThat(e.getMessage(), equalTo("Invalid detector rule: function freq_rare does not support rules with conditions")); + } + + public void testVerify_GivenTimeConditionRuleAndFunctionIsLatLong() { + Detector.Builder detector = new Detector.Builder("lat_long", "geo"); + detector.setRules(Collections.singletonList(new DetectionRule.Builder(Collections.singletonList( + new RuleCondition(RuleCondition.AppliesTo.TIME, Operator.GT, 42.0))).build())); + detector.build(); + } + + public void testVerify_GivenScopeRuleOnInvalidField() { + Detector.Builder detector = new Detector.Builder("mean", "my_metric"); + detector.setPartitionFieldName("my_partition"); + detector.setOverFieldName("my_over"); + detector.setByFieldName("my_by"); + + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder().exclude("my_metric", "my_filter_id")).build(); + detector.setRules(Collections.singletonList(rule)); + + ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, detector::build); + + assertEquals(Messages.getMessage(Messages.JOB_CONFIG_DETECTION_RULE_SCOPE_HAS_INVALID_FIELD, + "my_metric", "[my_by, my_over, my_partition]"), e.getMessage()); + } + + public void testVerify_GivenValidRule() { + Detector.Builder detector = new Detector.Builder("mean", "metricVale"); + detector.setByFieldName("metricName"); + detector.setPartitionFieldName("instance"); + DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(RuleConditionTests.createRandom())) + .setScope(RuleScope.builder() + .include("metricName", "f1") + .exclude("instance", "f2") + .build()) + .build(); + detector.setRules(Collections.singletonList(rule)); + detector.build(); + } + public void testExcludeFrequentForString() { assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("all")); assertEquals(Detector.ExcludeFrequent.ALL, Detector.ExcludeFrequent.forString("ALL")); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java new file mode 100644 index 0000000000000..241bf6593320c --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/FilterRefTests.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.job.config; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +public class FilterRefTests extends AbstractSerializingTestCase { + + @Override + protected FilterRef createTestInstance() { + return new FilterRef(randomAlphaOfLength(20), randomFrom(FilterRef.FilterType.values())); + } + + @Override + protected FilterRef doParseInstance(XContentParser parser) throws IOException { + return FilterRef.CONFIG_PARSER.parse(parser, null); + } + + @Override + protected Writeable.Reader instanceReader() { + return FilterRef::new; + } +} \ No newline at end of file diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 3663ff14e6302..c529d6ebfb368 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -53,10 +53,8 @@ protected JobUpdate createTestInstance() { List detectionRules = null; if (randomBoolean()) { detectionRules = new ArrayList<>(); - Condition condition = new Condition(Operator.GT, "5"); detectionRules.add(new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null))) - .setTargetFieldName("foo").build()); + Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5))).build()); } detectorUpdates.add(new JobUpdate.DetectorUpdate(i, detectorDescription, detectionRules)); } @@ -119,13 +117,11 @@ protected JobUpdate doParseInstance(XContentParser parser) { public void testMergeWithJob() { List detectorUpdates = new ArrayList<>(); List detectionRules1 = Collections.singletonList(new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5") - , null))) - .setTargetFieldName("mlcategory").build()); + Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5))) + .build()); detectorUpdates.add(new JobUpdate.DetectorUpdate(0, "description-1", detectionRules1)); List detectionRules2 = Collections.singletonList(new DetectionRule.Builder(Collections.singletonList( - new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, new Condition(Operator.GT, "5"), null))) - .setTargetFieldName("host").build()); + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5))).build()); detectorUpdates.add(new JobUpdate.DetectorUpdate(1, "description-2", detectionRules2)); ModelPlotConfig modelPlotConfig = new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java index 882c590983aae..07122818d5506 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleConditionTests.java @@ -5,36 +5,21 @@ */ package org.elasticsearch.xpack.core.ml.job.config; -import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.Writeable.Reader; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.ml.job.messages.Messages; public class RuleConditionTests extends AbstractSerializingTestCase { @Override protected RuleCondition createTestInstance() { - Condition condition = null; - String fieldName = null; - String valueFilter = null; - String fieldValue = null; - RuleConditionType type = randomFrom(RuleConditionType.values()); - if (type.isCategorical()) { - valueFilter = randomAlphaOfLengthBetween(1, 20); - if (randomBoolean()) { - fieldName = randomAlphaOfLengthBetween(1, 20); - } - } else { - // no need to randomize, it is properly randomly tested in - // ConditionTest - condition = new Condition(Operator.LT, Long.toString(randomLong())); - if (randomBoolean()) { - fieldName = randomAlphaOfLengthBetween(1, 20); - fieldValue = randomAlphaOfLengthBetween(1, 20); - } - } - return new RuleCondition(type, fieldName, fieldValue, condition, valueFilter); + return createRandom(); + } + + public static RuleCondition createRandom() { + RuleCondition.AppliesTo appliesTo = randomFrom(RuleCondition.AppliesTo.values()); + Operator operator = randomFrom(Operator.LT, Operator.LTE, Operator.GT, Operator.GTE); + return new RuleCondition(appliesTo, operator, randomDouble()); } @Override @@ -47,199 +32,52 @@ protected RuleCondition doParseInstance(XContentParser parser) { return RuleCondition.CONFIG_PARSER.apply(parser, null); } - public void testConstructor() { - RuleCondition condition = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueFilter"); - assertEquals(RuleConditionType.CATEGORICAL, condition.getType()); - assertNull(condition.getFieldName()); - assertNull(condition.getFieldValue()); - assertNull(condition.getCondition()); - } - public void testEqualsGivenSameObject() { - RuleCondition condition = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueFilter"); + RuleCondition condition = createRandom(); assertTrue(condition.equals(condition)); } public void testEqualsGivenString() { - assertFalse(new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "filter").equals("a string")); - } - - public void testEqualsGivenDifferentType() { - RuleCondition condition1 = createFullyPopulated(); - RuleCondition condition2 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "valueFilter"); - assertFalse(condition1.equals(condition2)); - assertFalse(condition2.equals(condition1)); - } - - public void testEqualsGivenDifferentFieldName() { - RuleCondition condition1 = createFullyPopulated(); - RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricNameaaa", "cpu", - new Condition(Operator.LT, "5"), null); - assertFalse(condition1.equals(condition2)); - assertFalse(condition2.equals(condition1)); - } - - public void testEqualsGivenDifferentFieldValue() { - RuleCondition condition1 = createFullyPopulated(); - RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpuaaa", - new Condition(Operator.LT, "5"), null); - assertFalse(condition1.equals(condition2)); - assertFalse(condition2.equals(condition1)); - } - - public void testEqualsGivenDifferentCondition() { - RuleCondition condition1 = createFullyPopulated(); - RuleCondition condition2 = new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpu", - new Condition(Operator.GT, "5"), null); - assertFalse(condition1.equals(condition2)); - assertFalse(condition2.equals(condition1)); - } - - public void testEqualsGivenDifferentValueFilter() { - RuleCondition condition1 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "myFilter"); - RuleCondition condition2 = new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, "myFilteraaa"); - assertFalse(condition1.equals(condition2)); - assertFalse(condition2.equals(condition1)); - } - - private static RuleCondition createFullyPopulated() { - return new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "cpu", new Condition(Operator.LT, "5"), null); - } - - public void testVerify_GivenCategoricalWithCondition() { - Condition condition = new Condition(Operator.MATCH, "text"); - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.CATEGORICAL, null, null, condition, null)); - assertEquals("Invalid detector rule: a categorical rule_condition does not support condition", e.getMessage()); - } - - public void testVerify_GivenCategoricalWithFieldValue() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.CATEGORICAL, "metric", "CPU", null, null)); - assertEquals("Invalid detector rule: a categorical rule_condition does not support field_value", e.getMessage()); - } - - public void testVerify_GivenCategoricalWithoutFilterId() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.CATEGORICAL, null, null, null, null)); - assertEquals("Invalid detector rule: a categorical rule_condition requires filter_id to be set", e.getMessage()); - } - - public void testVerify_GivenNumericalActualWithFilterId() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); - } - - public void testVerify_GivenNumericalActualWithoutCondition() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, null)); - assertEquals("Invalid detector rule: a numerical rule_condition requires condition to be set", e.getMessage()); + assertFalse(createRandom().equals("a string")); } - public void testVerify_GivenNumericalActualWithFieldNameButNoFieldValue() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", null, new Condition(Operator.LT, "5"), null)); - assertEquals("Invalid detector rule: a numerical rule_condition with field_name requires that field_value is set", e.getMessage()); - } - - public void testVerify_GivenNumericalTypicalWithFilterId() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); - } - - public void testVerify_GivenNumericalTypicalWithoutCondition() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, null, null)); - assertEquals("Invalid detector rule: a numerical rule_condition requires condition to be set", e.getMessage()); - } - - public void testVerify_GivenNumericalDiffAbsWithFilterId() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, "myFilter")); - assertEquals("Invalid detector rule: a numerical rule_condition does not support filter_id", e.getMessage()); - } - - public void testVerify_GivenNumericalDiffAbsWithoutCondition() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, null, null, null)); - assertEquals("Invalid detector rule: a numerical rule_condition requires condition to be set", e.getMessage()); - } - - public void testVerify_GivenFieldValueWithoutFieldName() { - Condition condition = new Condition(Operator.LTE, "5"); - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, null, "foo", condition, null)); - assertEquals("Invalid detector rule: missing field_name in rule_condition where field_value 'foo' is set", e.getMessage()); - } - - public void testVerify_GivenNumericalAndOperatorEquals() { - Condition condition = new Condition(Operator.EQ, "5"); - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); - assertEquals("Invalid detector rule: operator 'eq' is not allowed", e.getMessage()); - } - - public void testVerify_GivenNumericalAndOperatorMatch() { - Condition condition = new Condition(Operator.MATCH, "aaa"); - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition, null)); - assertEquals("Invalid detector rule: operator 'match' is not allowed", e.getMessage()); - } - - public void testVerify_GivenDetectionRuleWithInvalidCondition() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metricName", "CPU", new Condition(Operator.LT, "invalid"), - null)); - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, "invalid"), e.getMessage()); - } - - public void testVerify_GivenValidCategorical() { - // no validation error: - new RuleCondition(RuleConditionType.CATEGORICAL, "metric", null, null, "myFilter"); - new RuleCondition(RuleConditionType.CATEGORICAL_COMPLEMENT, "metric", null, null, "myFilter"); - } - - public void testVerify_GivenValidNumericalActual() { + public void testVerify_GivenValidActual() { // no validation error: - new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", "cpu", new Condition(Operator.GT, "5"), null); + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5.0); } - public void testVerify_GivenValidNumericalTypical() { + public void testVerify_GivenValidTypical() { // no validation error: - new RuleCondition(RuleConditionType.NUMERICAL_ACTUAL, "metric", "cpu", new Condition(Operator.GTE, "5"), null); + new RuleCondition(RuleCondition.AppliesTo.TYPICAL, Operator.GTE, 5.0); } - public void testVerify_GivenValidNumericalDiffAbs() { + public void testVerify_GivenValidDiffFromTypical() { // no validation error: - new RuleCondition(RuleConditionType.NUMERICAL_DIFF_ABS, "metric", "cpu", new Condition(Operator.LT, "5"), null); + new RuleCondition(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, Operator.LT, 5.0); } public void testCreateTimeBased() { RuleCondition timeBased = RuleCondition.createTime(Operator.GTE, 100L); - assertEquals(RuleConditionType.TIME, timeBased.getType()); - assertEquals(Operator.GTE, timeBased.getCondition().getOperator()); - assertEquals("100", timeBased.getCondition().getValue()); - assertNull(timeBased.getFieldName()); - assertNull(timeBased.getFieldValue()); - assertNull(timeBased.getFilterId()); - } - - public void testCreateTimeBased_GivenOperatorMatch() { - ElasticsearchException e = expectThrows(ElasticsearchException.class, - () -> RuleCondition.createTime(Operator.MATCH, 100L)); - assertEquals("Invalid detector rule: operator 'match' is not allowed", e.getMessage()); - } - - public void testCreateNumerical() { - RuleCondition ruleCondition = RuleCondition.createNumerical(RuleConditionType.NUMERICAL_ACTUAL, "foo", "bar", - new Condition(Operator.GTE, "100")); - assertEquals(RuleConditionType.NUMERICAL_ACTUAL, ruleCondition.getType()); - assertEquals(Operator.GTE, ruleCondition.getCondition().getOperator()); - assertEquals("100", ruleCondition.getCondition().getValue()); - assertEquals("foo", ruleCondition.getFieldName()); - assertEquals("bar", ruleCondition.getFieldValue()); - assertNull(ruleCondition.getFilterId()); + assertEquals(RuleCondition.AppliesTo.TIME, timeBased.getAppliesTo()); + assertEquals(Operator.GTE, timeBased.getOperator()); + assertEquals(100.0, timeBased.getValue(), 0.000001); + } + + public void testAppliesToFromString() { + assertEquals(RuleCondition.AppliesTo.ACTUAL, RuleCondition.AppliesTo.fromString("actual")); + assertEquals(RuleCondition.AppliesTo.ACTUAL, RuleCondition.AppliesTo.fromString("ACTUAL")); + assertEquals(RuleCondition.AppliesTo.TYPICAL, RuleCondition.AppliesTo.fromString("typical")); + assertEquals(RuleCondition.AppliesTo.TYPICAL, RuleCondition.AppliesTo.fromString("TYPICAL")); + assertEquals(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, RuleCondition.AppliesTo.fromString("diff_from_typical")); + assertEquals(RuleCondition.AppliesTo.DIFF_FROM_TYPICAL, RuleCondition.AppliesTo.fromString("DIFF_FROM_TYPICAL")); + assertEquals(RuleCondition.AppliesTo.TIME, RuleCondition.AppliesTo.fromString("time")); + assertEquals(RuleCondition.AppliesTo.TIME, RuleCondition.AppliesTo.fromString("TIME")); + } + + public void testAppliesToToString() { + assertEquals("actual", RuleCondition.AppliesTo.ACTUAL.toString()); + assertEquals("typical", RuleCondition.AppliesTo.TYPICAL.toString()); + assertEquals("diff_from_typical", RuleCondition.AppliesTo.DIFF_FROM_TYPICAL.toString()); + assertEquals("time", RuleCondition.AppliesTo.TIME.toString()); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java new file mode 100644 index 0000000000000..10b9c29aba7ef --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/RuleScopeTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.core.ml.job.config; + +import org.elasticsearch.ElasticsearchStatusException; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.util.set.Sets; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + +public class RuleScopeTests extends AbstractWireSerializingTestCase { + + @Override + protected RuleScope createTestInstance() { + RuleScope.Builder scope = RuleScope.builder(); + int count = randomIntBetween(0, 3); + for (int i = 0; i < count; ++i) { + if (randomBoolean()) { + scope.include(randomAlphaOfLength(20), randomAlphaOfLength(20)); + } else { + scope.exclude(randomAlphaOfLength(20), randomAlphaOfLength(20)); + } + } + return scope.build(); + } + + @Override + protected Writeable.Reader instanceReader() { + return RuleScope::new; + } + + public void testValidate_GivenEmpty() { + RuleScope scope = RuleScope.builder().build(); + assertThat(scope.isEmpty(), is(true)); + + scope.validate(Sets.newHashSet("a", "b")); + } + + public void testValidate_GivenMultipleValidFields() { + RuleScope scope = RuleScope.builder() + .include("foo", "filter1") + .exclude("bar", "filter2") + .include("foobar", "filter3") + .build(); + assertThat(scope.isEmpty(), is(false)); + + scope.validate(Sets.newHashSet("foo", "bar", "foobar")); + } + + public void testValidate_GivenMultipleFieldsIncludingInvalid() { + RuleScope scope = RuleScope.builder() + .include("foo", "filter1") + .exclude("bar", "filter2") + .include("foobar", "filter3") + .build(); + assertThat(scope.isEmpty(), is(false)); + + ElasticsearchStatusException e = expectThrows(ElasticsearchStatusException.class, + () -> scope.validate(Sets.newHashSet("foo", "foobar"))); + assertThat(e.getMessage(), equalTo("Invalid detector rule: scope field 'bar' is invalid; select from [foo, foobar]")); + } + + public void testGetReferencedFilters_GivenEmpty() { + assertThat(RuleScope.builder().build().getReferencedFilters().isEmpty(), is(true)); + } + + public void testGetReferencedFilters_GivenMultipleFields() { + RuleScope scope = RuleScope.builder() + .include("foo", "filter1") + .exclude("bar", "filter2") + .include("foobar", "filter3") + .build(); + assertThat(scope.getReferencedFilters(), contains("filter1", "filter2", "filter3")); + } +} \ No newline at end of file diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java index 92a0976f532b7..9b330e7165093 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/rollup/job/MetricsConfigSerializingTests.java @@ -24,7 +24,7 @@ public class MetricsConfigSerializingTests extends AbstractSerializingTestCase { @Override protected MetricConfig doParseInstance(XContentParser parser) throws IOException { - return MetricConfig.PARSER.apply(parser, null); + return MetricConfig.PARSER.apply(parser, null).build(); } @Override @@ -36,7 +36,7 @@ protected Writeable.Reader instanceReader() { protected MetricConfig createTestInstance() { return ConfigTestHelpers.getMetricConfig().build(); } - + public void testValidateNoMapping() throws IOException { ActionRequestValidationException e = new ActionRequestValidationException(); Map> responseMap = new HashMap<>(); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/ObjectCleanerThreadThreadFilter.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/ObjectCleanerThreadThreadFilter.java deleted file mode 100644 index e911920953ac2..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/ObjectCleanerThreadThreadFilter.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.core.test; - -import com.carrotsearch.randomizedtesting.ThreadFilter; - -/** - * The Netty object cleaner thread is not closeable and it does not terminate in a timely manner. This means that thread leak control in - * tests will fail test suites when the object cleaner thread has not terminated. Since there is not a reliable way to terminate this - * thread we instead filter it out of thread leak control. - */ -public class ObjectCleanerThreadThreadFilter implements ThreadFilter { - - @Override - public boolean reject(final Thread t) { - // TODO: replace with constant from Netty when https://github.com/netty/netty/pull/8014 is integrated - return "ObjectCleanerThread".equals(t.getName()); - } - -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/XPackIntegTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/XPackIntegTestCase.java deleted file mode 100644 index 87cf2d87f02a4..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/XPackIntegTestCase.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.core.test; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import org.elasticsearch.test.ESIntegTestCase; - -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) -public abstract class XPackIntegTestCase extends ESIntegTestCase { -} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/XPackSingleNodeTestCase.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/XPackSingleNodeTestCase.java deleted file mode 100644 index a3cbd875503f5..0000000000000 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/test/XPackSingleNodeTestCase.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.core.test; - -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; -import org.elasticsearch.test.ESSingleNodeTestCase; - -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) -public abstract class XPackSingleNodeTestCase extends ESSingleNodeTestCase { -} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index f783dfbddc35a..5de7962169279 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -54,6 +54,7 @@ import org.elasticsearch.xpack.core.ml.action.OpenJobAction; import org.elasticsearch.xpack.core.ml.action.PutJobAction; import org.elasticsearch.xpack.core.ml.action.UpdateJobAction; +import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskStatus; @@ -190,6 +191,14 @@ static PersistentTasksCustomMetaData.Assignment selectLeastLoadedMlNode(String j continue; } + if (jobHasRules(job) && node.getVersion().before(DetectionRule.VERSION_INTRODUCED)) { + String reason = "Not opening job [" + jobId + "] on node [" + nodeNameAndVersion(node) + "], because jobs using " + + "custom_rules require a node of version [" + DetectionRule.VERSION_INTRODUCED + "] or higher"; + logger.trace(reason); + reasons.add(reason); + continue; + } + long numberOfAssignedJobs = 0; int numberOfAllocatingJobs = 0; long assignedJobMemory = 0; @@ -373,6 +382,10 @@ private static boolean nodeSupportsModelSnapshotVersion(DiscoveryNode node, Job return node.getVersion().onOrAfter(job.getModelSnapshotMinVersion()); } + private static boolean jobHasRules(Job job) { + return job.getAnalysisConfig().getDetectors().stream().anyMatch(d -> d.getRules().isEmpty() == false); + } + static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion, Logger logger) throws IOException { List indicesToUpdate = new ArrayList<>(); @@ -646,9 +659,11 @@ public PersistentTasksCustomMetaData.Assignment getAssignment(OpenJobAction.JobP @Override public void validate(OpenJobAction.JobParams params, ClusterState clusterState) { + + TransportOpenJobAction.validate(params.getJobId(), MlMetadata.getMlMetadata(clusterState)); + // If we already know that we can't find an ml node because all ml nodes are running at capacity or // simply because there are no ml nodes in the cluster then we fail quickly here: - TransportOpenJobAction.validate(params.getJobId(), MlMetadata.getMlMetadata(clusterState)); PersistentTasksCustomMetaData.Assignment assignment = selectLeastLoadedMlNode(params.getJobId(), clusterState, maxConcurrentJobAllocations, fallbackMaxNumberOfOpenJobs, maxMachineMemoryPercent, logger); if (assignment.getExecutorNode() == null) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java index bed83ed82c1c9..3d261864ab409 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportStartDatafeedAction.java @@ -43,10 +43,12 @@ import org.elasticsearch.persistent.PersistentTasksExecutor; import org.elasticsearch.persistent.PersistentTasksService; import org.elasticsearch.xpack.ml.MachineLearning; +import org.elasticsearch.xpack.ml.datafeed.MlRemoteLicenseChecker; import org.elasticsearch.xpack.ml.datafeed.DatafeedManager; import org.elasticsearch.xpack.ml.datafeed.DatafeedNodeSelector; import org.elasticsearch.xpack.ml.datafeed.extractor.DataExtractorFactory; +import java.util.List; import java.util.Map; import java.util.function.Predicate; @@ -111,23 +113,25 @@ protected void masterOperation(StartDatafeedAction.Request request, ClusterState ActionListener listener) { StartDatafeedAction.DatafeedParams params = request.getParams(); if (licenseState.isMachineLearningAllowed()) { - ActionListener> finalListener = + + ActionListener> waitForTaskListener = new ActionListener>() { - @Override - public void onResponse(PersistentTasksCustomMetaData.PersistentTask persistentTask) { - waitForDatafeedStarted(persistentTask.getId(), params, listener); - } + @Override + public void onResponse(PersistentTasksCustomMetaData.PersistentTask + persistentTask) { + waitForDatafeedStarted(persistentTask.getId(), params, listener); + } - @Override - public void onFailure(Exception e) { - if (e instanceof ResourceAlreadyExistsException) { - logger.debug("datafeed already started", e); - e = new ElasticsearchStatusException("cannot start datafeed [" + params.getDatafeedId() + - "] because it has already been started", RestStatus.CONFLICT); - } - listener.onFailure(e); - } - }; + @Override + public void onFailure(Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + logger.debug("datafeed already started", e); + e = new ElasticsearchStatusException("cannot start datafeed [" + params.getDatafeedId() + + "] because it has already been started", RestStatus.CONFLICT); + } + listener.onFailure(e); + } + }; // Verify data extractor factory can be created, then start persistent task MlMetadata mlMetadata = MlMetadata.getMlMetadata(state); @@ -135,16 +139,39 @@ public void onFailure(Exception e) { validate(params.getDatafeedId(), mlMetadata, tasks); DatafeedConfig datafeed = mlMetadata.getDatafeed(params.getDatafeedId()); Job job = mlMetadata.getJobs().get(datafeed.getJobId()); - DataExtractorFactory.create(client, datafeed, job, ActionListener.wrap( - dataExtractorFactory -> - persistentTasksService.sendStartRequest(MLMetadataField.datafeedTaskId(params.getDatafeedId()), - StartDatafeedAction.TASK_NAME, params, finalListener) - , listener::onFailure)); + + if (MlRemoteLicenseChecker.containsRemoteIndex(datafeed.getIndices())) { + MlRemoteLicenseChecker remoteLicenseChecker = new MlRemoteLicenseChecker(client); + remoteLicenseChecker.checkRemoteClusterLicenses(MlRemoteLicenseChecker.remoteClusterNames(datafeed.getIndices()), + ActionListener.wrap( + response -> { + if (response.isViolated()) { + listener.onFailure(createUnlicensedError(datafeed.getId(), response)); + } else { + createDataExtractor(job, datafeed, params, waitForTaskListener); + } + }, + e -> listener.onFailure(createUnknownLicenseError(datafeed.getId(), + MlRemoteLicenseChecker.remoteIndices(datafeed.getIndices()), e)) + )); + } else { + createDataExtractor(job, datafeed, params, waitForTaskListener); + } } else { listener.onFailure(LicenseUtils.newComplianceException(XPackField.MACHINE_LEARNING)); } } + private void createDataExtractor(Job job, DatafeedConfig datafeed, StartDatafeedAction.DatafeedParams params, + ActionListener> + listener) { + DataExtractorFactory.create(client, datafeed, job, ActionListener.wrap( + dataExtractorFactory -> + persistentTasksService.sendStartRequest(MLMetadataField.datafeedTaskId(params.getDatafeedId()), + StartDatafeedAction.TASK_NAME, params, listener) + , listener::onFailure)); + } + @Override protected ClusterBlockException checkBlock(StartDatafeedAction.Request request, ClusterState state) { // We only delegate here to PersistentTasksService, but if there is a metadata writeblock, @@ -158,28 +185,29 @@ private void waitForDatafeedStarted(String taskId, StartDatafeedAction.DatafeedP DatafeedPredicate predicate = new DatafeedPredicate(); persistentTasksService.waitForPersistentTaskCondition(taskId, predicate, params.getTimeout(), new PersistentTasksService.WaitForPersistentTaskListener() { - @Override - public void onResponse(PersistentTasksCustomMetaData.PersistentTask persistentTask) { - if (predicate.exception != null) { - // We want to return to the caller without leaving an unassigned persistent task, to match - // what would have happened if the error had been detected in the "fast fail" validation - cancelDatafeedStart(persistentTask, predicate.exception, listener); - } else { - listener.onResponse(new StartDatafeedAction.Response(true)); - } - } + @Override + public void onResponse(PersistentTasksCustomMetaData.PersistentTask + persistentTask) { + if (predicate.exception != null) { + // We want to return to the caller without leaving an unassigned persistent task, to match + // what would have happened if the error had been detected in the "fast fail" validation + cancelDatafeedStart(persistentTask, predicate.exception, listener); + } else { + listener.onResponse(new StartDatafeedAction.Response(true)); + } + } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } - @Override - public void onTimeout(TimeValue timeout) { - listener.onFailure(new ElasticsearchException("Starting datafeed [" - + params.getDatafeedId() + "] timed out after [" + timeout + "]")); - } - }); + @Override + public void onTimeout(TimeValue timeout) { + listener.onFailure(new ElasticsearchException("Starting datafeed [" + + params.getDatafeedId() + "] timed out after [" + timeout + "]")); + } + }); } private void cancelDatafeedStart(PersistentTasksCustomMetaData.PersistentTask persistentTask, @@ -203,6 +231,25 @@ public void onFailure(Exception e) { ); } + private ElasticsearchStatusException createUnlicensedError(String datafeedId, + MlRemoteLicenseChecker.LicenseViolation licenseViolation) { + String message = "Cannot start datafeed [" + datafeedId + "] as it is configured to use " + + "indices on a remote cluster [" + licenseViolation.get().getClusterName() + + "] that is not licensed for Machine Learning. " + + MlRemoteLicenseChecker.buildErrorMessage(licenseViolation.get()); + + return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST); + } + + private ElasticsearchStatusException createUnknownLicenseError(String datafeedId, List remoteIndices, + Exception cause) { + String message = "Cannot start datafeed [" + datafeedId + "] as it is configured to use" + + " indices on a remote cluster " + remoteIndices + + " but the license type could not be verified"; + + return new ElasticsearchStatusException(message, RestStatus.BAD_REQUEST, new Exception(cause.getMessage())); + } + public static class StartDatafeedPersistentTasksExecutor extends PersistentTasksExecutor { private final DatafeedManager datafeedManager; private final IndexNameExpressionResolver resolver; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java index 37f9715d09464..0eb57ab79be5d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelector.java @@ -91,7 +91,7 @@ private AssignmentFailure verifyIndicesActive(DatafeedConfig datafeed) { List indices = datafeed.getIndices(); for (String index : indices) { - if (isRemoteIndex(index)) { + if (MlRemoteLicenseChecker.isRemoteIndex(index)) { // We cannot verify remote indices continue; } @@ -122,10 +122,6 @@ private AssignmentFailure verifyIndicesActive(DatafeedConfig datafeed) { return null; } - private boolean isRemoteIndex(String index) { - return index.indexOf(':') != -1; - } - private static class AssignmentFailure { private final String reason; private final boolean isCriticalForTaskCreation; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java new file mode 100644 index 0000000000000..b55713f6d0ab7 --- /dev/null +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java @@ -0,0 +1,192 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackInfoResponse; +import org.elasticsearch.transport.ActionNotFoundTransportException; +import org.elasticsearch.transport.RemoteClusterAware; +import org.elasticsearch.xpack.core.action.XPackInfoAction; +import org.elasticsearch.xpack.core.action.XPackInfoRequest; + +import java.util.EnumSet; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +/** + * ML datafeeds can use cross cluster search to access data in a remote cluster. + * The remote cluster should be licenced for ML this class performs that check + * using the _xpack (info) endpoint. + */ +public class MlRemoteLicenseChecker { + + private final Client client; + + public static class RemoteClusterLicenseInfo { + private final String clusterName; + private final XPackInfoResponse.LicenseInfo licenseInfo; + + RemoteClusterLicenseInfo(String clusterName, XPackInfoResponse.LicenseInfo licenseInfo) { + this.clusterName = clusterName; + this.licenseInfo = licenseInfo; + } + + public String getClusterName() { + return clusterName; + } + + public XPackInfoResponse.LicenseInfo getLicenseInfo() { + return licenseInfo; + } + } + + public class LicenseViolation { + private final RemoteClusterLicenseInfo licenseInfo; + + private LicenseViolation(@Nullable RemoteClusterLicenseInfo licenseInfo) { + this.licenseInfo = licenseInfo; + } + + public boolean isViolated() { + return licenseInfo != null; + } + + public RemoteClusterLicenseInfo get() { + return licenseInfo; + } + } + + public MlRemoteLicenseChecker(Client client) { + this.client = client; + } + + /** + * Check each cluster is licensed for ML. + * This function evaluates lazily and will terminate when the first cluster + * that is not licensed is found or an error occurs. + * + * @param clusterNames List of remote cluster names + * @param listener Response listener + */ + public void checkRemoteClusterLicenses(List clusterNames, ActionListener listener) { + final Iterator itr = clusterNames.iterator(); + if (itr.hasNext() == false) { + listener.onResponse(new LicenseViolation(null)); + return; + } + + final AtomicReference clusterName = new AtomicReference<>(itr.next()); + + ActionListener infoListener = new ActionListener() { + @Override + public void onResponse(XPackInfoResponse xPackInfoResponse) { + if (licenseSupportsML(xPackInfoResponse.getLicenseInfo()) == false) { + listener.onResponse(new LicenseViolation( + new RemoteClusterLicenseInfo(clusterName.get(), xPackInfoResponse.getLicenseInfo()))); + return; + } + + if (itr.hasNext()) { + clusterName.set(itr.next()); + remoteClusterLicense(clusterName.get(), this); + } else { + listener.onResponse(new LicenseViolation(null)); + } + } + + @Override + public void onFailure(Exception e) { + String message = "Could not determine the X-Pack licence type for cluster [" + clusterName.get() + "]"; + if (e instanceof ActionNotFoundTransportException) { + // This is likely to be because x-pack is not installed in the target cluster + message += ". Is X-Pack installed on the target cluster?"; + } + listener.onFailure(new ElasticsearchException(message, e)); + } + }; + + remoteClusterLicense(clusterName.get(), infoListener); + } + + private void remoteClusterLicense(String clusterName, ActionListener listener) { + Client remoteClusterClient = client.getRemoteClusterClient(clusterName); + ThreadContext threadContext = remoteClusterClient.threadPool().getThreadContext(); + try (ThreadContext.StoredContext ignore = threadContext.stashContext()) { + // we stash any context here since this is an internal execution and should not leak any + // existing context information. + threadContext.markAsSystemContext(); + + XPackInfoRequest request = new XPackInfoRequest(); + request.setCategories(EnumSet.of(XPackInfoRequest.Category.LICENSE)); + remoteClusterClient.execute(XPackInfoAction.INSTANCE, request, listener); + } + } + + static boolean licenseSupportsML(XPackInfoResponse.LicenseInfo licenseInfo) { + License.OperationMode mode = License.OperationMode.resolve(licenseInfo.getMode()); + return licenseInfo.getStatus() == License.Status.ACTIVE && + (mode == License.OperationMode.PLATINUM || mode == License.OperationMode.TRIAL); + } + + public static boolean isRemoteIndex(String index) { + return index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR) != -1; + } + + public static boolean containsRemoteIndex(List indices) { + return indices.stream().anyMatch(MlRemoteLicenseChecker::isRemoteIndex); + } + + /** + * Get any remote indices used in cross cluster search. + * Remote indices are of the form {@code cluster_name:index_name} + * @return List of remote cluster indices + */ + public static List remoteIndices(List indices) { + return indices.stream().filter(MlRemoteLicenseChecker::isRemoteIndex).collect(Collectors.toList()); + } + + /** + * Extract the list of remote cluster names from the list of indices. + * @param indices List of indices. Remote cluster indices are prefixed + * with {@code cluster-name:} + * @return Every cluster name found in {@code indices} + */ + public static List remoteClusterNames(List indices) { + return indices.stream() + .filter(MlRemoteLicenseChecker::isRemoteIndex) + .map(index -> index.substring(0, index.indexOf(RemoteClusterAware.REMOTE_CLUSTER_INDEX_SEPARATOR))) + .distinct() + .collect(Collectors.toList()); + } + + public static String buildErrorMessage(RemoteClusterLicenseInfo clusterLicenseInfo) { + StringBuilder error = new StringBuilder(); + if (clusterLicenseInfo.licenseInfo.getStatus() != License.Status.ACTIVE) { + error.append("The license on cluster [").append(clusterLicenseInfo.clusterName) + .append("] is not active. "); + } else { + License.OperationMode mode = License.OperationMode.resolve(clusterLicenseInfo.licenseInfo.getMode()); + if (mode != License.OperationMode.PLATINUM && mode != License.OperationMode.TRIAL) { + error.append("The license mode [").append(mode) + .append("] on cluster [") + .append(clusterLicenseInfo.clusterName) + .append("] does not enable Machine Learning. "); + } + } + + error.append(Strings.toString(clusterLicenseInfo.licenseInfo)); + return error.toString(); + } +} diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index 2042d917f6057..1fd73b96667b2 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -181,6 +181,7 @@ public void putJob(PutJobAction.Request request, AnalysisRegistry analysisRegist request.getJobBuilder().validateCategorizationAnalyzer(analysisRegistry, environment); Job job = request.getJobBuilder().build(new Date()); + if (job.getDataDescription() != null && job.getDataDescription().getFormat() == DataDescription.DataFormat.DELIMITED) { DEPRECATION_LOGGER.deprecated("Creating jobs with delimited data format is deprecated. Please use xcontent instead."); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java index 049880b1ac224..21be815d561a8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcess.java @@ -117,7 +117,7 @@ void writeUpdateDetectorRulesMessage(int detectorIndex, List rule /** * Ask the job to start persisting model state in the background - * @throws IOException + * @throws IOException If writing the request fails */ void persistJob() throws IOException; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java index 5ee07d63a968b..6ef2d92d9c7c6 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/action/TransportOpenJobActionTests.java @@ -36,9 +36,15 @@ import org.elasticsearch.xpack.core.ml.MlMetaIndex; import org.elasticsearch.xpack.core.ml.MlMetadata; import org.elasticsearch.xpack.core.ml.action.OpenJobAction; +import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; +import org.elasticsearch.xpack.core.ml.job.config.DataDescription; +import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; +import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.JobTaskStatus; +import org.elasticsearch.xpack.core.ml.job.config.Operator; +import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndexFields; import org.elasticsearch.xpack.core.ml.job.persistence.ElasticsearchMappings; @@ -49,6 +55,7 @@ import java.io.IOException; import java.net.InetAddress; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.HashMap; @@ -427,6 +434,62 @@ public void testSelectLeastLoadedMlNode_noNodesMatchingModelSnapshotMinVersion() assertNull(result.getExecutorNode()); } + public void testSelectLeastLoadedMlNode_jobWithRulesButNoNodeMeetsRequiredVersion() { + Map nodeAttr = new HashMap<>(); + nodeAttr.put(MachineLearning.ML_ENABLED_NODE_ATTR, "true"); + DiscoveryNodes nodes = DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + nodeAttr, Collections.emptySet(), Version.V_6_2_0)) + .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), + nodeAttr, Collections.emptySet(), Version.V_6_3_0)) + .build(); + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + addJobTask("job_with_rules", "_node_id1", null, tasksBuilder); + PersistentTasksCustomMetaData tasks = tasksBuilder.build(); + + ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); + MetaData.Builder metaData = MetaData.builder(); + RoutingTable.Builder routingTable = RoutingTable.builder(); + addJobAndIndices(metaData, routingTable, jobWithRulesCreator(), "job_with_rules"); + cs.nodes(nodes); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); + cs.metaData(metaData); + cs.routingTable(routingTable.build()); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", cs.build(), + 2, 10, 30, logger); + assertThat(result.getExplanation(), containsString( + "because jobs using custom_rules require a node of version [6.4.0] or higher")); + assertNull(result.getExecutorNode()); + } + + public void testSelectLeastLoadedMlNode_jobWithRulesAndNodeMeetsRequiredVersion() { + Map nodeAttr = new HashMap<>(); + nodeAttr.put(MachineLearning.ML_ENABLED_NODE_ATTR, "true"); + DiscoveryNodes nodes = DiscoveryNodes.builder() + .add(new DiscoveryNode("_node_name1", "_node_id1", new TransportAddress(InetAddress.getLoopbackAddress(), 9300), + nodeAttr, Collections.emptySet(), Version.V_6_2_0)) + .add(new DiscoveryNode("_node_name2", "_node_id2", new TransportAddress(InetAddress.getLoopbackAddress(), 9301), + nodeAttr, Collections.emptySet(), Version.V_6_4_0)) + .build(); + + PersistentTasksCustomMetaData.Builder tasksBuilder = PersistentTasksCustomMetaData.builder(); + addJobTask("job_with_rules", "_node_id1", null, tasksBuilder); + PersistentTasksCustomMetaData tasks = tasksBuilder.build(); + + ClusterState.Builder cs = ClusterState.builder(new ClusterName("_name")); + MetaData.Builder metaData = MetaData.builder(); + RoutingTable.Builder routingTable = RoutingTable.builder(); + addJobAndIndices(metaData, routingTable, jobWithRulesCreator(), "job_with_rules"); + cs.nodes(nodes); + metaData.putCustom(PersistentTasksCustomMetaData.TYPE, tasks); + cs.metaData(metaData); + cs.routingTable(routingTable.build()); + Assignment result = TransportOpenJobAction.selectLeastLoadedMlNode("job_with_rules", cs.build(), + 2, 10, 30, logger); + assertNotNull(result.getExecutorNode()); + } + public void testVerifyIndicesPrimaryShardsAreActive() { MetaData.Builder metaData = MetaData.builder(); RoutingTable.Builder routingTable = RoutingTable.builder(); @@ -645,4 +708,21 @@ private ClusterState getClusterStateWithMappingsWithMetaData(Map return csBuilder.build(); } + private static Function jobWithRulesCreator() { + return jobId -> { + DetectionRule rule = new DetectionRule.Builder(Arrays.asList( + new RuleCondition(RuleCondition.AppliesTo.TYPICAL, Operator.LT, 100.0) + )).build(); + + Detector.Builder detector = new Detector.Builder("count", null); + detector.setRules(Arrays.asList(rule)); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Arrays.asList(detector.build())); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + Job.Builder job = new Job.Builder(jobId); + job.setAnalysisConfig(analysisConfig); + job.setDataDescription(dataDescription); + return job.build(new Date()); + }; + } + } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java new file mode 100644 index 0000000000000..47d4d30a7c6e4 --- /dev/null +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java @@ -0,0 +1,200 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.ml.datafeed; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.client.Client; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.license.License; +import org.elasticsearch.license.XPackInfoResponse; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xpack.core.action.XPackInfoAction; +import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.is; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.same; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +public class MlRemoteLicenseCheckerTests extends ESTestCase { + + public void testIsRemoteIndex() { + List indices = Arrays.asList("local-index1", "local-index2"); + assertFalse(MlRemoteLicenseChecker.containsRemoteIndex(indices)); + indices = Arrays.asList("local-index1", "remote-cluster:remote-index2"); + assertTrue(MlRemoteLicenseChecker.containsRemoteIndex(indices)); + } + + public void testRemoteIndices() { + List indices = Collections.singletonList("local-index"); + assertThat(MlRemoteLicenseChecker.remoteIndices(indices), is(empty())); + indices = Arrays.asList("local-index", "remote-cluster:index1", "local-index2", "remote-cluster2:index1"); + assertThat(MlRemoteLicenseChecker.remoteIndices(indices), containsInAnyOrder("remote-cluster:index1", "remote-cluster2:index1")); + } + + public void testRemoteClusterNames() { + List indices = Arrays.asList("local-index1", "local-index2"); + assertThat(MlRemoteLicenseChecker.remoteClusterNames(indices), empty()); + indices = Arrays.asList("local-index1", "remote-cluster1:remote-index2"); + assertThat(MlRemoteLicenseChecker.remoteClusterNames(indices), contains("remote-cluster1")); + indices = Arrays.asList("remote-cluster1:index2", "index1", "remote-cluster2:index1"); + assertThat(MlRemoteLicenseChecker.remoteClusterNames(indices), contains("remote-cluster1", "remote-cluster2")); + indices = Arrays.asList("remote-cluster1:index2", "index1", "remote-cluster2:index1", "remote-cluster2:index2"); + assertThat(MlRemoteLicenseChecker.remoteClusterNames(indices), contains("remote-cluster1", "remote-cluster2")); + } + + public void testLicenseSupportsML() { + XPackInfoResponse.LicenseInfo licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "trial", "trial", + License.Status.ACTIVE, randomNonNegativeLong()); + assertTrue(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); + + licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "trial", "trial", License.Status.EXPIRED, randomNonNegativeLong()); + assertFalse(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); + + licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "GOLD", "GOLD", License.Status.ACTIVE, randomNonNegativeLong()); + assertFalse(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); + + licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", License.Status.ACTIVE, randomNonNegativeLong()); + assertTrue(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); + } + + public void testCheckRemoteClusterLicenses_givenValidLicenses() { + final AtomicInteger index = new AtomicInteger(0); + final List responses = new ArrayList<>(); + + Client client = createMockClient(); + doAnswer(invocationMock -> { + @SuppressWarnings("raw_types") + ActionListener listener = (ActionListener) invocationMock.getArguments()[2]; + listener.onResponse(responses.get(index.getAndIncrement())); + return null; + }).when(client).execute(same(XPackInfoAction.INSTANCE), any(), any()); + + + List remoteClusterNames = Arrays.asList("valid1", "valid2", "valid3"); + responses.add(new XPackInfoResponse(null, createPlatinumLicenseResponse(), null)); + responses.add(new XPackInfoResponse(null, createPlatinumLicenseResponse(), null)); + responses.add(new XPackInfoResponse(null, createPlatinumLicenseResponse(), null)); + + MlRemoteLicenseChecker licenseChecker = new MlRemoteLicenseChecker(client); + AtomicReference licCheckResponse = new AtomicReference<>(); + + licenseChecker.checkRemoteClusterLicenses(remoteClusterNames, + new ActionListener() { + @Override + public void onResponse(MlRemoteLicenseChecker.LicenseViolation response) { + licCheckResponse.set(response); + } + + @Override + public void onFailure(Exception e) { + fail(e.getMessage()); + } + }); + + verify(client, times(3)).execute(same(XPackInfoAction.INSTANCE), any(), any()); + assertNotNull(licCheckResponse.get()); + assertFalse(licCheckResponse.get().isViolated()); + assertNull(licCheckResponse.get().get()); + } + + public void testCheckRemoteClusterLicenses_givenInvalidLicense() { + final AtomicInteger index = new AtomicInteger(0); + List remoteClusterNames = Arrays.asList("good", "cluster-with-basic-license", "good2"); + final List responses = new ArrayList<>(); + responses.add(new XPackInfoResponse(null, createPlatinumLicenseResponse(), null)); + responses.add(new XPackInfoResponse(null, createBasicLicenseResponse(), null)); + responses.add(new XPackInfoResponse(null, createPlatinumLicenseResponse(), null)); + + Client client = createMockClient(); + doAnswer(invocationMock -> { + @SuppressWarnings("raw_types") + ActionListener listener = (ActionListener) invocationMock.getArguments()[2]; + listener.onResponse(responses.get(index.getAndIncrement())); + return null; + }).when(client).execute(same(XPackInfoAction.INSTANCE), any(), any()); + + MlRemoteLicenseChecker licenseChecker = new MlRemoteLicenseChecker(client); + AtomicReference licCheckResponse = new AtomicReference<>(); + + licenseChecker.checkRemoteClusterLicenses(remoteClusterNames, + new ActionListener() { + @Override + public void onResponse(MlRemoteLicenseChecker.LicenseViolation response) { + licCheckResponse.set(response); + } + + @Override + public void onFailure(Exception e) { + fail(e.getMessage()); + } + }); + + verify(client, times(2)).execute(same(XPackInfoAction.INSTANCE), any(), any()); + assertNotNull(licCheckResponse.get()); + assertTrue(licCheckResponse.get().isViolated()); + assertEquals("cluster-with-basic-license", licCheckResponse.get().get().getClusterName()); + assertEquals("BASIC", licCheckResponse.get().get().getLicenseInfo().getType()); + } + + public void testBuildErrorMessage() { + XPackInfoResponse.LicenseInfo platinumLicence = createPlatinumLicenseResponse(); + MlRemoteLicenseChecker.RemoteClusterLicenseInfo info = + new MlRemoteLicenseChecker.RemoteClusterLicenseInfo("platinum-cluster", platinumLicence); + assertEquals(Strings.toString(platinumLicence), MlRemoteLicenseChecker.buildErrorMessage(info)); + + XPackInfoResponse.LicenseInfo basicLicense = createBasicLicenseResponse(); + info = new MlRemoteLicenseChecker.RemoteClusterLicenseInfo("basic-cluster", basicLicense); + String expected = "The license mode [BASIC] on cluster [basic-cluster] does not enable Machine Learning. " + + Strings.toString(basicLicense); + assertEquals(expected, MlRemoteLicenseChecker.buildErrorMessage(info)); + + XPackInfoResponse.LicenseInfo expiredLicense = createExpiredLicenseResponse(); + info = new MlRemoteLicenseChecker.RemoteClusterLicenseInfo("expired-cluster", expiredLicense); + expected = "The license on cluster [expired-cluster] is not active. " + Strings.toString(expiredLicense); + assertEquals(expected, MlRemoteLicenseChecker.buildErrorMessage(info)); + } + + private Client createMockClient() { + Client client = mock(Client.class); + ThreadPool threadPool = mock(ThreadPool.class); + when(client.threadPool()).thenReturn(threadPool); + when(threadPool.getThreadContext()).thenReturn(new ThreadContext(Settings.EMPTY)); + when(client.getRemoteClusterClient(anyString())).thenReturn(client); + return client; + } + + private XPackInfoResponse.LicenseInfo createPlatinumLicenseResponse() { + return new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", License.Status.ACTIVE, randomNonNegativeLong()); + } + + private XPackInfoResponse.LicenseInfo createBasicLicenseResponse() { + return new XPackInfoResponse.LicenseInfo("uid", "BASIC", "BASIC", License.Status.ACTIVE, randomNonNegativeLong()); + } + + private XPackInfoResponse.LicenseInfo createExpiredLicenseResponse() { + return new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", License.Status.EXPIRED, randomNonNegativeLong()); + } +} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java index 120f04e95e70b..7e0dc453f07ee 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/integration/JobProviderIT.java @@ -27,29 +27,28 @@ import org.elasticsearch.xpack.core.ml.calendars.Calendar; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.Connective; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.config.RuleAction; -import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; +import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; -import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; -import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder; -import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; +import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCountsTests; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSizeStats; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.Quantiles; import org.elasticsearch.xpack.ml.LocalStateMachineLearning; import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.ml.MlSingleNodeTestCase; +import org.elasticsearch.xpack.ml.job.persistence.CalendarQueryBuilder; import org.elasticsearch.xpack.ml.job.persistence.JobDataCountsPersister; import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import org.elasticsearch.xpack.ml.job.persistence.JobResultsPersister; -import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCountsTests; +import org.elasticsearch.xpack.ml.job.persistence.ScheduledEventsQueryBuilder; +import org.elasticsearch.xpack.ml.job.process.autodetect.params.AutodetectParams; import org.junit.Before; import java.io.IOException; @@ -511,20 +510,15 @@ private Job.Builder createJob(String jobId, List filterIds, List private AnalysisConfig.Builder createAnalysisConfig(List filterIds) { Detector.Builder detector = new Detector.Builder("mean", "field"); detector.setByFieldName("by_field"); + List rules = new ArrayList<>(); - if (!filterIds.isEmpty()) { - List conditions = new ArrayList<>(); - - for (String filterId : filterIds) { - conditions.add(RuleCondition.createCategorical("by_field", filterId)); - } - - DetectionRule.Builder rule = new DetectionRule.Builder(conditions) - .setActions(RuleAction.FILTER_RESULTS) - .setConditionsConnective(Connective.OR); + for (String filterId : filterIds) { + RuleScope.Builder ruleScope = RuleScope.builder(); + ruleScope.include("by_field", filterId); - detector.setRules(Collections.singletonList(rule.build())); + rules.add(new DetectionRule.Builder(ruleScope).setActions(RuleAction.SKIP_RESULT).build()); } + detector.setRules(rules); return new AnalysisConfig.Builder(Collections.singletonList(detector.build())); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java index 64b3bfbab45c8..454f941d6c8b0 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/JobManagerTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; import org.elasticsearch.index.analysis.AnalysisRegistry; +import org.elasticsearch.persistent.PersistentTasksCustomMetaData; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.MLMetadataField; import org.elasticsearch.xpack.core.ml.MachineLearningField; @@ -32,8 +33,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; -import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; -import org.elasticsearch.persistent.PersistentTasksCustomMetaData; +import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests; import org.elasticsearch.xpack.ml.job.persistence.JobProvider; import org.elasticsearch.xpack.ml.job.process.autodetect.UpdateParams; @@ -177,9 +177,7 @@ public void onFailure(Exception e) { public void testUpdateProcessOnFilterChanged() { Detector.Builder detectorReferencingFilter = new Detector.Builder("count", null); detectorReferencingFilter.setByFieldName("foo"); - RuleCondition.createCategorical("foo", "foo_filter"); - DetectionRule filterRule = new DetectionRule.Builder(Collections.singletonList( - RuleCondition.createCategorical("foo", "foo_filter"))).build(); + DetectionRule filterRule = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "foo_filter")).build(); detectorReferencingFilter.setRules(Collections.singletonList(filterRule)); AnalysisConfig.Builder filterAnalysisConfig = new AnalysisConfig.Builder(Collections.singletonList( detectorReferencingFilter.build())); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/ConditionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/ConditionTests.java deleted file mode 100644 index 09bae9fb8cb9b..0000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/ConditionTests.java +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.config; - -import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.common.io.stream.Writeable.Reader; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.job.config.Condition; -import org.elasticsearch.xpack.core.ml.job.config.Operator; -import org.elasticsearch.xpack.core.ml.job.messages.Messages; - -import java.io.IOException; - -public class ConditionTests extends AbstractSerializingTestCase { - - public void testSetValues() { - Condition cond = new Condition(Operator.EQ, "5"); - assertEquals(Operator.EQ, cond.getOperator()); - assertEquals("5", cond.getValue()); - } - - public void testHashCodeAndEquals() { - Condition cond1 = new Condition(Operator.MATCH, "regex"); - Condition cond2 = new Condition(Operator.MATCH, "regex"); - - assertEquals(cond1, cond2); - assertEquals(cond1.hashCode(), cond2.hashCode()); - - Condition cond3 = new Condition(Operator.EQ, "5"); - assertFalse(cond1.equals(cond3)); - assertFalse(cond1.hashCode() == cond3.hashCode()); - } - - @Override - protected Condition createTestInstance() { - Operator op = randomFrom(Operator.values()); - Condition condition; - switch (op) { - case EQ: - case GT: - case GTE: - case LT: - case LTE: - condition = new Condition(op, Double.toString(randomDouble())); - break; - case MATCH: - condition = new Condition(op, randomAlphaOfLengthBetween(1, 20)); - break; - default: - throw new AssertionError("Unknown operator selected: " + op); - } - return condition; - } - - @Override - protected Reader instanceReader() { - return Condition::new; - } - - @Override - protected Condition doParseInstance(XContentParser parser) { - return Condition.PARSER.apply(parser, null); - } - - public void testVerifyArgsNumericArgs() { - new Condition(Operator.LTE, "100"); - new Condition(Operator.GT, "10.0"); - } - - public void testVerify_GivenEmptyValue() { - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, () -> new Condition(Operator.LT, "")); - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NUMBER, ""), e.getMessage()); - } - - public void testVerify_GivenInvalidRegex() { - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, () -> new Condition(Operator.MATCH, "[*")); - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_REGEX, "[*"), e.getMessage()); - } - - public void testVerify_GivenNullRegex() { - ElasticsearchException e = ESTestCase.expectThrows(ElasticsearchException.class, - () -> new Condition(Operator.MATCH, null)); - assertEquals(Messages.getMessage(Messages.JOB_CONFIG_CONDITION_INVALID_VALUE_NULL, "[*"), e.getMessage()); - } - - @Override - protected Condition mutateInstance(Condition instance) throws IOException { - Operator op = instance.getOperator(); - String value = instance.getValue(); - switch (between(0, 1)) { - case 0: - Operator newOp = op; - while (newOp == op) { - newOp = randomFrom(Operator.values()); - } - if (op == Operator.MATCH && newOp != Operator.MATCH) { - value = Double.toString(randomDouble()); - } - op = newOp; - break; - case 1: - value = Double.toString(randomDouble()); - break; - default: - throw new AssertionError("Illegal randomisation branch"); - } - return new Condition(op, value); - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/ConnectiveTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/ConnectiveTests.java deleted file mode 100644 index c8f9b6d36cbca..0000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/ConnectiveTests.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.config; - -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.job.config.Connective; - -import java.io.IOException; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class ConnectiveTests extends ESTestCase { - - public void testForString() { - assertEquals(Connective.OR, Connective.fromString("or")); - assertEquals(Connective.OR, Connective.fromString("OR")); - assertEquals(Connective.AND, Connective.fromString("and")); - assertEquals(Connective.AND, Connective.fromString("AND")); - } - - public void testToString() { - assertEquals("or", Connective.OR.toString()); - assertEquals("and", Connective.AND.toString()); - } - - public void testValidOrdinals() { - assertThat(Connective.OR.ordinal(), equalTo(0)); - assertThat(Connective.AND.ordinal(), equalTo(1)); - } - - public void testwriteTo() throws Exception { - try (BytesStreamOutput out = new BytesStreamOutput()) { - Connective.OR.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(0)); - } - } - - try (BytesStreamOutput out = new BytesStreamOutput()) { - Connective.AND.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(1)); - } - } - } - - public void testReadFrom() throws Exception { - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(0); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(Connective.readFromStream(in), equalTo(Connective.OR)); - } - } - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(1); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(Connective.readFromStream(in), equalTo(Connective.AND)); - } - } - } - - public void testInvalidReadFrom() throws Exception { - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(randomIntBetween(2, Integer.MAX_VALUE)); - try (StreamInput in = out.bytes().streamInput()) { - Connective.readFromStream(in); - fail("Expected IOException"); - } catch (IOException e) { - assertThat(e.getMessage(), containsString("Unknown Connective ordinal [")); - } - } - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/OperatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/OperatorTests.java index 07f4d4c5edfa2..469dbf822e005 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/OperatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/OperatorTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.xpack.core.ml.job.config.Operator; import java.io.IOException; -import java.util.regex.Pattern; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -19,110 +18,63 @@ public class OperatorTests extends ESTestCase { public void testFromString() { - assertEquals(Operator.fromString("eq"), Operator.EQ); assertEquals(Operator.fromString("gt"), Operator.GT); assertEquals(Operator.fromString("gte"), Operator.GTE); assertEquals(Operator.fromString("lte"), Operator.LTE); assertEquals(Operator.fromString("lt"), Operator.LT); - assertEquals(Operator.fromString("match"), Operator.MATCH); assertEquals(Operator.fromString("Gt"), Operator.GT); - assertEquals(Operator.fromString("EQ"), Operator.EQ); assertEquals(Operator.fromString("GTE"), Operator.GTE); - assertEquals(Operator.fromString("Match"), Operator.MATCH); } public void testToString() { - assertEquals("eq", Operator.EQ.toString()); assertEquals("gt", Operator.GT.toString()); assertEquals("gte", Operator.GTE.toString()); assertEquals("lte", Operator.LTE.toString()); assertEquals("lt", Operator.LT.toString()); - assertEquals("match", Operator.MATCH.toString()); } public void testTest() { - assertTrue(Operator.GT.expectsANumericArgument()); assertTrue(Operator.GT.test(1.0, 0.0)); assertFalse(Operator.GT.test(0.0, 1.0)); - assertTrue(Operator.GTE.expectsANumericArgument()); assertTrue(Operator.GTE.test(1.0, 0.0)); assertTrue(Operator.GTE.test(1.0, 1.0)); assertFalse(Operator.GTE.test(0.0, 1.0)); - assertTrue(Operator.EQ.expectsANumericArgument()); - assertTrue(Operator.EQ.test(0.0, 0.0)); - assertFalse(Operator.EQ.test(1.0, 0.0)); - - assertTrue(Operator.LT.expectsANumericArgument()); assertTrue(Operator.LT.test(0.0, 1.0)); assertFalse(Operator.LT.test(0.0, 0.0)); - assertTrue(Operator.LTE.expectsANumericArgument()); assertTrue(Operator.LTE.test(0.0, 1.0)); assertTrue(Operator.LTE.test(1.0, 1.0)); assertFalse(Operator.LTE.test(1.0, 0.0)); } - public void testMatch() { - assertFalse(Operator.MATCH.expectsANumericArgument()); - assertFalse(Operator.MATCH.test(0.0, 1.0)); - - Pattern pattern = Pattern.compile("^aa.*"); - - assertTrue(Operator.MATCH.match(pattern, "aaaaa")); - assertFalse(Operator.MATCH.match(pattern, "bbaaa")); - } - - public void testValidOrdinals() { - assertThat(Operator.EQ.ordinal(), equalTo(0)); - assertThat(Operator.GT.ordinal(), equalTo(1)); - assertThat(Operator.GTE.ordinal(), equalTo(2)); - assertThat(Operator.LT.ordinal(), equalTo(3)); - assertThat(Operator.LTE.ordinal(), equalTo(4)); - assertThat(Operator.MATCH.ordinal(), equalTo(5)); - } - - public void testwriteTo() throws Exception { - try (BytesStreamOutput out = new BytesStreamOutput()) { - Operator.EQ.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(0)); - } - } - + public void testWriteTo() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { Operator.GT.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(1)); + assertThat(in.readVInt(), equalTo(0)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { Operator.GTE.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(2)); + assertThat(in.readVInt(), equalTo(1)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { Operator.LT.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(3)); + assertThat(in.readVInt(), equalTo(2)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { Operator.LTE.writeTo(out); try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(4)); - } - } - - try (BytesStreamOutput out = new BytesStreamOutput()) { - Operator.MATCH.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(5)); + assertThat(in.readVInt(), equalTo(3)); } } } @@ -130,40 +82,28 @@ public void testwriteTo() throws Exception { public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(Operator.readFromStream(in), equalTo(Operator.EQ)); - } - } - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(1); try (StreamInput in = out.bytes().streamInput()) { assertThat(Operator.readFromStream(in), equalTo(Operator.GT)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(2); + out.writeVInt(1); try (StreamInput in = out.bytes().streamInput()) { assertThat(Operator.readFromStream(in), equalTo(Operator.GTE)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(3); + out.writeVInt(2); try (StreamInput in = out.bytes().streamInput()) { assertThat(Operator.readFromStream(in), equalTo(Operator.LT)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(4); + out.writeVInt(3); try (StreamInput in = out.bytes().streamInput()) { assertThat(Operator.readFromStream(in), equalTo(Operator.LTE)); } } - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(5); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(Operator.readFromStream(in), equalTo(Operator.MATCH)); - } - } } public void testInvalidReadFrom() throws Exception { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java index 1aeefc47ac61c..fadf81693934b 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleActionTests.java @@ -15,27 +15,29 @@ public class RuleActionTests extends ESTestCase { public void testForString() { - assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("filter_results")); - assertEquals(RuleAction.FILTER_RESULTS, RuleAction.fromString("FILTER_RESULTS")); - assertEquals(RuleAction.SKIP_SAMPLING, RuleAction.fromString("SKip_sampLing")); + assertEquals(RuleAction.SKIP_RESULT, RuleAction.fromString("skip_result")); + assertEquals(RuleAction.SKIP_RESULT, RuleAction.fromString("SKIP_RESULT")); + assertEquals(RuleAction.SKIP_MODEL_UPDATE, RuleAction.fromString("skip_model_update")); + assertEquals(RuleAction.SKIP_MODEL_UPDATE, RuleAction.fromString("SKIP_MODEL_UPDATE")); } public void testToString() { - assertEquals("filter_results", RuleAction.FILTER_RESULTS.toString()); + assertEquals("skip_result", RuleAction.SKIP_RESULT.toString()); + assertEquals("skip_model_update", RuleAction.SKIP_MODEL_UPDATE.toString()); } public void testReadFrom() throws Exception { try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(0); try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.FILTER_RESULTS)); + assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_RESULT)); } } try (BytesStreamOutput out = new BytesStreamOutput()) { out.writeVInt(1); try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_SAMPLING)); + assertThat(RuleAction.readFromStream(in), equalTo(RuleAction.SKIP_MODEL_UPDATE)); } } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTypeTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTypeTests.java deleted file mode 100644 index bec7ca24fbe68..0000000000000 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/config/RuleConditionTypeTests.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.xpack.ml.job.config; - -import org.elasticsearch.common.io.stream.BytesStreamOutput; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.ml.job.config.RuleConditionType; - -import java.io.IOException; -import java.util.EnumSet; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; - -public class RuleConditionTypeTests extends ESTestCase { - - public void testFromString() { - assertEquals(RuleConditionType.CATEGORICAL, RuleConditionType.fromString("categorical")); - assertEquals(RuleConditionType.CATEGORICAL, RuleConditionType.fromString("CATEGORICAL")); - assertEquals(RuleConditionType.NUMERICAL_ACTUAL, RuleConditionType.fromString("numerical_actual")); - assertEquals(RuleConditionType.NUMERICAL_ACTUAL, RuleConditionType.fromString("NUMERICAL_ACTUAL")); - assertEquals(RuleConditionType.NUMERICAL_TYPICAL, RuleConditionType.fromString("numerical_typical")); - assertEquals(RuleConditionType.NUMERICAL_TYPICAL, RuleConditionType.fromString("NUMERICAL_TYPICAL")); - assertEquals(RuleConditionType.NUMERICAL_DIFF_ABS, RuleConditionType.fromString("numerical_diff_abs")); - assertEquals(RuleConditionType.NUMERICAL_DIFF_ABS, RuleConditionType.fromString("NUMERICAL_DIFF_ABS")); - } - - public void testToString() { - assertEquals("categorical", RuleConditionType.CATEGORICAL.toString()); - assertEquals("categorical_complement", RuleConditionType.CATEGORICAL_COMPLEMENT.toString()); - assertEquals("numerical_actual", RuleConditionType.NUMERICAL_ACTUAL.toString()); - assertEquals("numerical_typical", RuleConditionType.NUMERICAL_TYPICAL.toString()); - assertEquals("numerical_diff_abs", RuleConditionType.NUMERICAL_DIFF_ABS.toString()); - } - - public void testValidOrdinals() { - assertThat(RuleConditionType.CATEGORICAL.ordinal(), equalTo(0)); - assertThat(RuleConditionType.NUMERICAL_ACTUAL.ordinal(), equalTo(1)); - assertThat(RuleConditionType.NUMERICAL_TYPICAL.ordinal(), equalTo(2)); - assertThat(RuleConditionType.NUMERICAL_DIFF_ABS.ordinal(), equalTo(3)); - } - - public void testwriteTo() throws Exception { - try (BytesStreamOutput out = new BytesStreamOutput()) { - RuleConditionType.CATEGORICAL.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(0)); - } - } - - try (BytesStreamOutput out = new BytesStreamOutput()) { - RuleConditionType.NUMERICAL_ACTUAL.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(1)); - } - } - - try (BytesStreamOutput out = new BytesStreamOutput()) { - RuleConditionType.NUMERICAL_TYPICAL.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(2)); - } - } - - try (BytesStreamOutput out = new BytesStreamOutput()) { - RuleConditionType.NUMERICAL_DIFF_ABS.writeTo(out); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(in.readVInt(), equalTo(3)); - } - } - } - - public void testReadFrom() throws Exception { - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(0); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.CATEGORICAL)); - } - } - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(1); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_ACTUAL)); - } - } - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(2); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_TYPICAL)); - } - } - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(3); - try (StreamInput in = out.bytes().streamInput()) { - assertThat(RuleConditionType.readFromStream(in), equalTo(RuleConditionType.NUMERICAL_DIFF_ABS)); - } - } - } - - public void testInvalidReadFrom() throws Exception { - try (BytesStreamOutput out = new BytesStreamOutput()) { - out.writeVInt(randomIntBetween(4, Integer.MAX_VALUE)); - try (StreamInput in = out.bytes().streamInput()) { - RuleConditionType.readFromStream(in); - fail("Expected IOException"); - } catch (IOException e) { - assertThat(e.getMessage(), containsString("Unknown RuleConditionType ordinal [")); - } - } - } - - public void testIsNumerical() { - for (RuleConditionType type : EnumSet.allOf(RuleConditionType.class)) { - boolean isNumerical = type.isNumerical(); - if (type == RuleConditionType.NUMERICAL_ACTUAL || - type == RuleConditionType.NUMERICAL_DIFF_ABS || - type == RuleConditionType.NUMERICAL_TYPICAL) { - assertTrue(isNumerical); - } else { - assertFalse(isNumerical); - } - } - } - - public void testIsCategorical() { - for (RuleConditionType type : EnumSet.allOf(RuleConditionType.class)) { - boolean isCategorical = type.isCategorical(); - if (type == RuleConditionType.CATEGORICAL || - type == RuleConditionType.CATEGORICAL_COMPLEMENT) { - assertTrue(isCategorical); - } else { - assertFalse(isCategorical); - } - } - } -} diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java index 0aecad2b3a634..cedc65c2ee225 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectCommunicatorTests.java @@ -22,7 +22,7 @@ import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobUpdate; -import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; +import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.core.ml.job.process.autodetect.output.FlushAcknowledgement; import org.elasticsearch.xpack.ml.job.categorization.CategorizationAnalyzerTests; import org.elasticsearch.xpack.ml.job.persistence.StateStreamer; @@ -91,10 +91,7 @@ public void testWriteUpdateProcessMessage() throws IOException { when(process.isReady()).thenReturn(true); AutodetectCommunicator communicator = createAutodetectCommunicator(process, mock(AutoDetectResultProcessor.class)); - List conditions = Collections.singletonList( - RuleCondition.createCategorical("foo", "bar")); - - DetectionRule updatedRule = new DetectionRule.Builder(conditions).build(); + DetectionRule updatedRule = new DetectionRule.Builder(RuleScope.builder().exclude("foo", "bar")).build(); List detectorUpdates = Collections.singletonList( new JobUpdate.DetectorUpdate(0, "updated description", Collections.singletonList(updatedRule))); diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java index 130476951abbd..8c32a5bb40d46 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ControlMsgToProcessWriterTests.java @@ -8,18 +8,16 @@ import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; -import org.elasticsearch.xpack.core.ml.job.config.Condition; -import org.elasticsearch.xpack.core.ml.job.config.Connective; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.config.ModelPlotConfig; import org.elasticsearch.xpack.core.ml.job.config.Operator; import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; -import org.elasticsearch.xpack.core.ml.job.config.RuleConditionType; import org.elasticsearch.xpack.ml.job.process.autodetect.params.DataLoadParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.FlushJobParams; import org.elasticsearch.xpack.ml.job.process.autodetect.params.TimeRange; import org.junit.Before; +import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import org.mockito.Mockito; @@ -31,6 +29,7 @@ import java.util.Optional; import java.util.stream.IntStream; +import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -190,22 +189,18 @@ public void testWriteUpdateModelPlotMessage() throws IOException { public void testWriteUpdateDetectorRulesMessage() throws IOException { ControlMsgToProcessWriter writer = new ControlMsgToProcessWriter(lengthEncodedWriter, 4); - DetectionRule rule1 = new DetectionRule.Builder(createRule("5")).setTargetFieldName("targetField1") - .setTargetFieldValue("targetValue").setConditionsConnective(Connective.AND).build(); - DetectionRule rule2 = new DetectionRule.Builder(createRule("5")).setTargetFieldName("targetField2") - .setTargetFieldValue("targetValue").setConditionsConnective(Connective.AND).build(); + DetectionRule rule1 = new DetectionRule.Builder(createRule(5)).build(); + DetectionRule rule2 = new DetectionRule.Builder(createRule(5)).build(); writer.writeUpdateDetectorRulesMessage(2, Arrays.asList(rule1, rule2)); InOrder inOrder = inOrder(lengthEncodedWriter); inOrder.verify(lengthEncodedWriter).writeNumFields(4); inOrder.verify(lengthEncodedWriter, times(3)).writeField(""); inOrder.verify(lengthEncodedWriter).writeField("u[detectorRules]\ndetectorIndex=2\n" + - "rulesJson=[{\"actions\":[\"filter_results\"],\"conditions_connective\":\"and\",\"conditions\":" + - "[{\"type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," + - "\"target_field_name\":\"targetField1\",\"target_field_value\":\"targetValue\"}," + - "{\"actions\":[\"filter_results\"],\"conditions_connective\":\"and\",\"conditions\":[" + - "{\"type\":\"numerical_actual\",\"condition\":{\"operator\":\"gt\",\"value\":\"5\"}}]," + - "\"target_field_name\":\"targetField2\",\"target_field_value\":\"targetValue\"}]"); + "rulesJson=[{\"actions\":[\"skip_result\"],\"conditions\":" + + "[{\"applies_to\":\"actual\",\"operator\":\"gt\",\"value\":5.0}]}," + + "{\"actions\":[\"skip_result\"],\"conditions\":[" + + "{\"applies_to\":\"actual\",\"operator\":\"gt\",\"value\":5.0}]}]"); verifyNoMoreInteractions(lengthEncodedWriter); } @@ -244,16 +239,17 @@ public void testWriteUpdateScheduledEventsMessage() throws IOException { InOrder inOrder = inOrder(lengthEncodedWriter); inOrder.verify(lengthEncodedWriter).writeNumFields(2); inOrder.verify(lengthEncodedWriter, times(1)).writeField(""); - inOrder.verify(lengthEncodedWriter).writeField("u[scheduledEvents]\n" + ArgumentCaptor capturedMessage = ArgumentCaptor.forClass(String.class); + inOrder.verify(lengthEncodedWriter).writeField(capturedMessage.capture()); + assertThat(capturedMessage.getValue(), equalTo("u[scheduledEvents]\n" + "scheduledevent.0.description = new year\n" - + "scheduledevent.0.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," - + "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1514764800\"}}," - + "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1514851200\"}}]}]\n" + + "scheduledevent.0.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"]," + + "\"conditions\":[{\"applies_to\":\"time\",\"operator\":\"gte\",\"value\":1.5147648E9}," + + "{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5148512E9}]}]\n" + "scheduledevent.1.description = Jan maintenance day\n" - + "scheduledevent.1.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," - + "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1515196800\"}}," - + "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1515283200\"}}]}]\n" - ); + + "scheduledevent.1.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"]," + + "\"conditions\":[{\"applies_to\":\"time\",\"operator\":\"gte\",\"value\":1.5151968E9}," + + "{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5152832E9}]}]\n")); verifyNoMoreInteractions(lengthEncodedWriter); } @@ -288,8 +284,7 @@ public void testWriteStartBackgroundPersistMessage() throws IOException { verifyNoMoreInteractions(lengthEncodedWriter); } - private static List createRule(String value) { - Condition condition = new Condition(Operator.GT, value); - return Collections.singletonList(RuleCondition.createNumerical(RuleConditionType.NUMERICAL_ACTUAL, null, null, condition)); + private static List createRule(double value) { + return Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, value)); } } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java index 57917f6761bb7..bf08d09bf090c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/FieldConfigWriterTests.java @@ -10,16 +10,14 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.core.ml.calendars.ScheduledEvent; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.Condition; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.config.Operator; import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; -import org.elasticsearch.xpack.core.ml.job.config.RuleConditionType; +import org.elasticsearch.xpack.ml.MachineLearning; import org.ini4j.Config; import org.ini4j.Ini; import org.ini4j.Profile.Section; @@ -193,9 +191,8 @@ public void testWrite_GivenDetectorWithRules() throws IOException { Detector.Builder detector = new Detector.Builder("mean", "metricValue"); detector.setByFieldName("metricName"); detector.setPartitionFieldName("instance"); - RuleCondition ruleCondition = RuleCondition.createNumerical - (RuleConditionType.NUMERICAL_ACTUAL, "metricName", "metricValue", new Condition(Operator.LT, "5")); - DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).setTargetFieldName("instance").build(); + RuleCondition ruleCondition = new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.LT, 5); + DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(ruleCondition)).build(); detector.setRules(Collections.singletonList(rule)); AnalysisConfig.Builder builder = new AnalysisConfig.Builder(Collections.singletonList(detector.build())); @@ -255,14 +252,12 @@ public void testWrite_GivenScheduledEvents() throws IOException { verify(writer).write("detector.0.clause = count\n" + "scheduledevent.0.description = The Ashes\n" + - "scheduledevent.0.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," + - "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1511395200\"}}," + - "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1515369600\"}}]}]\n" + + "scheduledevent.0.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"],\"conditions\":[{\"applies_to\":\"time\"," + + "\"operator\":\"gte\",\"value\":1.5113952E9},{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5153696E9}]}]\n" + "scheduledevent.1.description = elasticon\n" + - "scheduledevent.1.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," + - "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1519603200\"}}," + - "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1519862400\"}}]}]" + - "\n"); + "scheduledevent.1.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"]," + + "\"conditions\":[{\"applies_to\":\"time\",\"operator\":\"gte\",\"value\":1.5196032E9}," + + "{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5198624E9}]}]\n"); verifyNoMoreInteractions(writer); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ScheduledEventsWriterTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ScheduledEventsWriterTests.java index ae1b77b34089c..2806aef128575 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ScheduledEventsWriterTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/job/process/autodetect/writer/ScheduledEventsWriterTests.java @@ -44,13 +44,13 @@ public void testWrite() throws IOException { new ScheduledEventsWriter(events, TimeValue.timeValueHours(1), buffer).write(); String expectedString = "scheduledevent.0.description = Black Friday\n" + - "scheduledevent.0.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," + - "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1511395200\"}}," + - "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1515369600\"}}]}]\n" + + "scheduledevent.0.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"]," + + "\"conditions\":[{\"applies_to\":\"time\",\"operator\":\"gte\",\"value\":1.5113952E9}," + + "{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5153696E9}]}]\n" + "scheduledevent.1.description = Blue Monday\n" + - "scheduledevent.1.rules = [{\"actions\":[\"filter_results\",\"skip_sampling\"],\"conditions_connective\":\"and\"," + - "\"conditions\":[{\"type\":\"time\",\"condition\":{\"operator\":\"gte\",\"value\":\"1519603200\"}}," + - "{\"type\":\"time\",\"condition\":{\"operator\":\"lt\",\"value\":\"1519862400\"}}]}]" + + "scheduledevent.1.rules = [{\"actions\":[\"skip_result\",\"skip_model_update\"]," + + "\"conditions\":[{\"applies_to\":\"time\",\"operator\":\"gte\",\"value\":1.5196032E9}," + + "{\"applies_to\":\"time\",\"operator\":\"lt\",\"value\":1.5198624E9}]}]" + "\n"; assertThat(buffer.toString(), equalTo(expectedString)); } diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java index 29c0ef2516553..7a1ca6a00f46c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/support/BaseMlIntegTestCase.java @@ -27,17 +27,19 @@ import org.elasticsearch.test.MockHttpTransport; import org.elasticsearch.test.discovery.TestZenDiscovery; import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; +import org.elasticsearch.xpack.core.ml.action.GetJobsAction; +import org.elasticsearch.xpack.core.ml.action.util.QueryPage; +import org.elasticsearch.xpack.core.ml.client.MachineLearningClient; +import org.elasticsearch.xpack.ml.LocalStateMachineLearning; +import org.elasticsearch.xpack.ml.MachineLearning; import org.elasticsearch.xpack.core.ml.MachineLearningField; import org.elasticsearch.xpack.core.ml.action.CloseJobAction; import org.elasticsearch.xpack.core.ml.action.DeleteDatafeedAction; import org.elasticsearch.xpack.core.ml.action.DeleteJobAction; -import org.elasticsearch.xpack.core.ml.action.GetDatafeedsAction; import org.elasticsearch.xpack.core.ml.action.GetDatafeedsStatsAction; -import org.elasticsearch.xpack.core.ml.action.GetJobsAction; import org.elasticsearch.xpack.core.ml.action.GetJobsStatsAction; import org.elasticsearch.xpack.core.ml.action.StopDatafeedAction; -import org.elasticsearch.xpack.core.ml.action.util.QueryPage; -import org.elasticsearch.xpack.core.ml.client.MachineLearningClient; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedConfig; import org.elasticsearch.xpack.core.ml.datafeed.DatafeedState; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; @@ -47,9 +49,6 @@ import org.elasticsearch.xpack.core.ml.job.config.Job; import org.elasticsearch.xpack.core.ml.job.config.JobState; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.DataCounts; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; -import org.elasticsearch.xpack.ml.LocalStateMachineLearning; -import org.elasticsearch.xpack.ml.MachineLearning; import org.junit.After; import org.junit.Before; @@ -70,7 +69,7 @@ */ @ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0, supportsDedicatedMasters = false) -public abstract class BaseMlIntegTestCase extends XPackIntegTestCase { +public abstract class BaseMlIntegTestCase extends ESIntegTestCase { @Override protected boolean ignoreExternalCluster() { diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java index f918c7aaf568c..4d398a401abe8 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/action/TransportMonitoringBulkActionTests.java @@ -7,10 +7,13 @@ import java.util.concurrent.ExecutorService; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.support.ActionFilter; import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.ActionTestUtils; import org.elasticsearch.cluster.ClusterName; import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MetaData; @@ -49,7 +52,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.concurrent.ExecutionException; import static org.elasticsearch.Version.CURRENT; import static org.hamcrest.Matchers.containsString; @@ -119,7 +121,7 @@ public void testExecuteWithGlobalBlock() throws Exception { monitoringService); final MonitoringBulkRequest request = randomRequest(); - final ExecutionException e = expectThrows(ExecutionException.class, () -> action.execute(request).get()); + final ClusterBlockException e = expectThrows(ClusterBlockException.class, () -> ActionTestUtils.executeBlocking(action, request)); assertThat(e, hasToString(containsString("ClusterBlockException[blocked by: [SERVICE_UNAVAILABLE/2/no master]"))); } @@ -138,7 +140,7 @@ public void testExecuteIgnoresRequestWhenCollectionIsDisabled() throws Exception final MonitoringBulkRequest request = new MonitoringBulkRequest(); request.add(doc); - final MonitoringBulkResponse response = action.execute(request).get(); + final MonitoringBulkResponse response = ActionTestUtils.executeBlocking(action, request); assertThat(response.status(), is(RestStatus.OK)); assertThat(response.isIgnored(), is(true)); @@ -155,13 +157,14 @@ public void testExecuteEmptyRequest() { monitoringService); final MonitoringBulkRequest request = new MonitoringBulkRequest(); - final ExecutionException e = expectThrows(ExecutionException.class, () -> action.execute(request).get()); + final ActionRequestValidationException e = expectThrows(ActionRequestValidationException.class, + () -> ActionTestUtils.executeBlocking(action, request)); assertThat(e, hasToString(containsString("no monitoring documents added"))); } @SuppressWarnings("unchecked") - public void testExecuteRequest() throws Exception { + public void testExecuteRequest() { when(monitoringService.isMonitoringActive()).thenReturn(true); final DiscoveryNode discoveryNode = new DiscoveryNode("_id", new TransportAddress(TransportAddress.META_ADDRESS, 9300), CURRENT); @@ -217,7 +220,7 @@ public void testExecuteRequest() throws Exception { final TransportMonitoringBulkAction action = new TransportMonitoringBulkAction(Settings.EMPTY, threadPool, clusterService, transportService, filters, resolver, exporters, monitoringService); - action.execute(request).get(); + ActionTestUtils.executeBlocking(action, request); verify(threadPool).executor(ThreadPool.Names.GENERIC); verify(exporters).export(any(Collection.class), any(ActionListener.class)); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java index d2e359990966b..664745b19204b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java @@ -115,7 +115,6 @@ import org.elasticsearch.xpack.core.security.authz.RoleDescriptor; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; import org.elasticsearch.xpack.core.security.authz.accesscontrol.SecurityIndexSearcherWrapper; -import org.elasticsearch.xpack.core.security.authz.accesscontrol.SetSecurityUserProcessor; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissions; import org.elasticsearch.xpack.core.security.authz.permission.FieldPermissionsCache; import org.elasticsearch.xpack.security.authz.store.FileRolesStore; @@ -174,6 +173,7 @@ import org.elasticsearch.xpack.security.authz.accesscontrol.OptOutQueryCache; import org.elasticsearch.xpack.security.authz.store.CompositeRolesStore; import org.elasticsearch.xpack.security.authz.store.NativeRolesStore; +import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor; import org.elasticsearch.xpack.security.rest.SecurityRestFilter; import org.elasticsearch.xpack.security.rest.action.RestAuthenticateAction; import org.elasticsearch.xpack.security.rest.action.oauth2.RestGetTokenAction; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SetSecurityUserProcessor.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java similarity index 98% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SetSecurityUserProcessor.java rename to x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java index 051a077646320..15ac88b4d9462 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SetSecurityUserProcessor.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessor.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.core.security.authz.accesscontrol; +package org.elasticsearch.xpack.security.ingest; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.ingest.AbstractProcessor; diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java index 95af766515777..2170c55ee0192 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContext.java @@ -145,7 +145,7 @@ public void closeChannel() { selector.queueWrite(writeOperation); return; } - selector.queueWriteInChannelBuffer(writeOperation); + selector.writeToChannel(writeOperation); } } diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index f56affec02be0..857c2f6e472d5 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -23,8 +23,6 @@ grant codeBase "${codebase.xmlsec-2.0.8.jar}" { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - - permission java.lang.RuntimePermission "setContextClassLoader"; }; grant codeBase "${codebase.netty-transport}" { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java index 20e2f904323dc..815f26942767a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecurityIntegTestCase.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.test; import io.netty.util.ThreadDeathWatcher; @@ -42,8 +41,8 @@ import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; import org.elasticsearch.xpack.security.LocalStateSecurity; + import org.elasticsearch.xpack.security.support.SecurityIndexManager; import org.junit.AfterClass; import org.junit.Before; @@ -76,7 +75,7 @@ * * @see SecuritySettingsSource */ -public abstract class SecurityIntegTestCase extends XPackIntegTestCase { +public abstract class SecurityIntegTestCase extends ESIntegTestCase { private static SecuritySettingsSource SECURITY_DEFAULT_SETTINGS; protected static SecureString BOOTSTRAP_PASSWORD = null; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java index 1c6f8e847d39c..cda627806e7b5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/test/SecuritySingleNodeTestCase.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.test; import io.netty.util.ThreadDeathWatcher; @@ -23,7 +22,6 @@ import org.elasticsearch.license.LicenseService; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.plugins.PluginInfo; -import org.elasticsearch.xpack.core.test.XPackSingleNodeTestCase; import org.elasticsearch.xpack.security.LocalStateSecurity; import org.junit.AfterClass; import org.junit.Before; @@ -51,7 +49,7 @@ * {@link SecurityIntegTestCase} due to simplicity and improved speed from not needing to start * multiple nodes and wait for the cluster to form. */ -public abstract class SecuritySingleNodeTestCase extends XPackSingleNodeTestCase { +public abstract class SecuritySingleNodeTestCase extends ESSingleNodeTestCase { private static SecuritySettingsSource SECURITY_DEFAULT_SETTINGS = null; private static CustomSecuritySettingsSource customSecuritySettingsSource = null; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SetSecurityUserProcessorFactoryTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java similarity index 94% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SetSecurityUserProcessorFactoryTests.java rename to x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java index 483d3de2beaa3..19da9b18ea650 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/accesscontrol/SetSecurityUserProcessorFactoryTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorFactoryTests.java @@ -3,11 +3,11 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.core.security.authz.accesscontrol; +package org.elasticsearch.xpack.security.ingest; import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.security.authz.accesscontrol.SetSecurityUserProcessor.Property; +import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor.Property; import java.util.Arrays; import java.util.EnumSet; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SetSecurityUserProcessorTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java similarity index 97% rename from x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SetSecurityUserProcessorTests.java rename to x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java index c9ef169a375a5..26c59a1ef5470 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/SetSecurityUserProcessorTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/ingest/SetSecurityUserProcessorTests.java @@ -3,7 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -package org.elasticsearch.xpack.security.authz.accesscontrol; +package org.elasticsearch.xpack.security.ingest; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -11,9 +11,8 @@ import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.core.security.authc.Authentication; import org.elasticsearch.xpack.core.security.authc.AuthenticationField; -import org.elasticsearch.xpack.core.security.authz.accesscontrol.SetSecurityUserProcessor; -import org.elasticsearch.xpack.core.security.authz.accesscontrol.SetSecurityUserProcessor.Property; import org.elasticsearch.xpack.core.security.user.User; +import org.elasticsearch.xpack.security.ingest.SetSecurityUserProcessor.Property; import java.util.Collections; import java.util.EnumSet; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java index 14a22d300d12d..bfee50b65bff4 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/transport/nio/SSLChannelContextTests.java @@ -345,7 +345,7 @@ public void testInitiateCloseFromSameThreadSchedulesCloseNotify() { context.closeChannel(); ArgumentCaptor captor = ArgumentCaptor.forClass(WriteOperation.class); - verify(selector).queueWriteInChannelBuffer(captor.capture()); + verify(selector).writeToChannel(captor.capture()); context.queueWriteOperation(captor.getValue()); verify(sslDriver).initiateClose(); diff --git a/x-pack/plugin/sql/build.gradle b/x-pack/plugin/sql/build.gradle index c52413aa4e1d6..8b406235985c6 100644 --- a/x-pack/plugin/sql/build.gradle +++ b/x-pack/plugin/sql/build.gradle @@ -5,7 +5,7 @@ esplugin { name 'x-pack-sql' description 'The Elasticsearch plugin that powers SQL for Elasticsearch' classname 'org.elasticsearch.xpack.sql.plugin.SqlPlugin' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'lang-painless'] } configurations { @@ -20,6 +20,7 @@ integTest.enabled = false dependencies { compileOnly "org.elasticsearch.plugin:x-pack-core:${version}" + compileOnly project(':modules:lang-painless') compile project('sql-proto') compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}" compile "org.antlr:antlr4-runtime:4.5.3" diff --git a/x-pack/plugin/sql/licenses/antlr4-runtime-4.5.3.jar.sha1 b/x-pack/plugin/sql/licenses/antlr4-runtime-4.5.3.jar.sha1 deleted file mode 100644 index 535955b7d6826..0000000000000 --- a/x-pack/plugin/sql/licenses/antlr4-runtime-4.5.3.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2609e36f18f7e8d593cc1cddfb2ac776dc96b8e0 \ No newline at end of file diff --git a/x-pack/plugin/sql/licenses/antlr4-runtime-LICENSE.txt b/x-pack/plugin/sql/licenses/antlr4-runtime-LICENSE.txt deleted file mode 100644 index 95d0a2554f686..0000000000000 --- a/x-pack/plugin/sql/licenses/antlr4-runtime-LICENSE.txt +++ /dev/null @@ -1,26 +0,0 @@ -[The "BSD license"] -Copyright (c) 2015 Terence Parr, Sam Harwell -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. The name of the author may not be used to endorse or promote products - derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/x-pack/plugin/sql/licenses/antlr4-runtime-NOTICE.txt b/x-pack/plugin/sql/licenses/antlr4-runtime-NOTICE.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java index 77a6db3009d0c..606728222787b 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/datetime/DateTimeFunction.java @@ -51,8 +51,18 @@ public abstract class DateTimeFunction extends UnaryScalarFunction { protected final NodeInfo info() { return NodeInfo.create(this, ctorForInfo(), field(), timeZone()); } + protected abstract NodeInfo.NodeCtor2 ctorForInfo(); + @Override + protected TypeResolution resolveType() { + if (field().dataType() == DataType.DATE) { + return TypeResolution.TYPE_RESOLVED; + } + return new TypeResolution("Function [" + functionName() + "] cannot be applied on a non-date expression ([" + + Expressions.name(field()) + "] of type [" + field().dataType().esType + "])"); + } + public TimeZone timeZone() { return timeZone; } @@ -69,18 +79,12 @@ public Object fold() { return null; } - ZonedDateTime time = ZonedDateTime.ofInstant( - Instant.ofEpochMilli(folded.getMillis()), ZoneId.of(timeZone.getID())); - return time.get(chronoField()); + return dateTimeChrono(folded.getMillis(), timeZone.getID(), chronoField().name()); } - @Override - protected TypeResolution resolveType() { - if (field().dataType() == DataType.DATE) { - return TypeResolution.TYPE_RESOLVED; - } - return new TypeResolution("Function [" + functionName() + "] cannot be applied on a non-date expression ([" - + Expressions.name(field()) + "] of type [" + field().dataType().esType + "])"); + public static Integer dateTimeChrono(long millis, String tzId, String chronoName) { + ZonedDateTime time = ZonedDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneId.of(tzId)); + return Integer.valueOf(time.get(ChronoField.valueOf(chronoName))); } @Override @@ -88,28 +92,11 @@ protected ScriptTemplate asScriptFrom(FieldAttribute field) { ParamsBuilder params = paramsBuilder(); String template = null; - if (TimeZone.getTimeZone("UTC").equals(timeZone)) { - // TODO: it would be nice to be able to externalize the extract function and reuse the script across all extractors - template = formatTemplate("doc[{}].value.get" + extractFunction() + "()"); - params.variable(field.name()); - } else { - // TODO ewwww - /* - * This uses the Java 8 time API because Painless doesn't whitelist creation of new - * Joda classes. - * - * The actual script is - * ZonedDateTime.ofInstant(Instant.ofEpochMilli(.value.millis), - * ZoneId.of()).get(ChronoField.get(MONTH_OF_YEAR)) - */ - - template = formatTemplate("ZonedDateTime.ofInstant(Instant.ofEpochMilli(doc[{}].value.millis), " - + "ZoneId.of({})).get(ChronoField.valueOf({}))"); - params.variable(field.name()) - .variable(timeZone.getID()) - .variable(chronoField().name()); - } - + template = formatTemplate("{sql}.dateTimeChrono(doc[{}].value.millis, {}, {})"); + params.variable(field.name()) + .variable(timeZone.getID()) + .variable(chronoField().name()); + return new ScriptTemplate(template, params.build(), dataType()); } @@ -119,10 +106,6 @@ protected ScriptTemplate asScriptFrom(AggregateFunctionAttribute aggregate) { throw new UnsupportedOperationException(); } - protected String extractFunction() { - return getClass().getSimpleName(); - } - /** * Used for generating the painless script version of this function when the time zone is not UTC */ @@ -164,4 +147,4 @@ public boolean equals(Object obj) { public int hashCode() { return Objects.hash(field(), timeZone); } -} +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java index 323ccc4e072c6..35b7680dcca78 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/script/ScriptTemplate.java @@ -7,6 +7,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptType; +import org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalSqlScriptUtils; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.util.StringUtils; @@ -92,6 +93,6 @@ public String toString() { } public static String formatTemplate(String template) { - return template.replace("{}", "params.%s"); + return template.replace("{sql}", InternalSqlScriptUtils.class.getSimpleName()).replace("{}", "params.%s"); } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java new file mode 100644 index 0000000000000..802aa4a7c09bb --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.whitelist; + +import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction; + +/** + * Whitelisted class for SQL scripts. + * Acts as a registry of the various static methods used internally by the scalar functions + * (to simplify the whitelist definition). + */ +public final class InternalSqlScriptUtils { + + private InternalSqlScriptUtils() {} + + public static Integer dateTimeChrono(long millis, String tzId, String chronoName) { + return DateTimeFunction.dateTimeChrono(millis, tzId, chronoName); + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPainlessExtension.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPainlessExtension.java new file mode 100644 index 0000000000000..426d725ac79d8 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plugin/SqlPainlessExtension.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.plugin; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.FilterScript; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.SearchScript; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.singletonList; + +public class SqlPainlessExtension implements PainlessExtension { + + private static final Whitelist WHITELIST = WhitelistLoader.loadFromResourceFiles(SqlPainlessExtension.class, "sql_whitelist.txt"); + + @Override + public Map, List> getContextWhitelists() { + Map, List> whitelist = new HashMap<>(); + List list = singletonList(WHITELIST); + whitelist.put(FilterScript.CONTEXT, list); + whitelist.put(SearchScript.AGGS_CONTEXT, list); + whitelist.put(SearchScript.CONTEXT, list); + whitelist.put(SearchScript.SCRIPT_SORT_CONTEXT, list); + return whitelist; + } +} diff --git a/x-pack/plugin/sql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/sql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..5f2f571f015d9 --- /dev/null +++ b/x-pack/plugin/sql/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.sql.plugin.SqlPainlessExtension \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt new file mode 100644 index 0000000000000..8dae4f8c0d1d6 --- /dev/null +++ b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt @@ -0,0 +1,12 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# This file contains a whitelist for SQL specific utilities available inside SQL scripting + +class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalSqlScriptUtils { + + Integer dateTimeChrono(long, String, String) +} \ No newline at end of file diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml index 5203ead3ce8a9..d3165260f4b95 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/filter_crud.yml @@ -152,15 +152,11 @@ setup: "analysis_config" : { "bucket_span": "3600s", "detectors" :[{"function":"mean","field_name":"responsetime", "by_field_name": "airline", - "rules": [ + "custom_rules": [ { - "conditions": [ - { - "type": "categorical", - "field_name": "airline", - "filter_id": "filter-foo" - } - ] + "scope": { + "airline": {"filter_id": "filter-foo"} + } } ]}] }, diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml index ed6b9ead1a3b1..df505176ae739 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml @@ -301,10 +301,26 @@ { "groups": ["group-1", "group-2"], "description":"Post update description", - "detectors": [{"detector_index": 0, "rules": {"target_field_name": "airline", - "conditions": [ { "type": "numerical_actual", - "condition": {"operator": "gt", "value": "10" } } ] } }, - {"detector_index": 1, "description": "updated description"}], + "detectors": [ + { + "detector_index": 0, + "custom_rules":[ + { + "conditions": [ + { + "applies_to": "actual", + "operator": "gt", + "value": 10 + } + ] + } + ] + }, + { + "detector_index": 1, + "description": "updated description" + } + ], "model_plot_config": { "enabled": false, "terms": "foobar" @@ -324,7 +340,8 @@ - match: { model_plot_config.enabled: false } - match: { model_plot_config.terms: "foobar" } - match: { analysis_config.categorization_filters: ["cat3.*"] } - - match: { analysis_config.detectors.0.rules.0.target_field_name: "airline" } + - match: { analysis_config.detectors.0.custom_rules.0.actions: ["skip_result"] } + - length: { analysis_config.detectors.0.custom_rules.0.conditions: 1 } - match: { analysis_config.detectors.0.detector_index: 0 } - match: { analysis_config.detectors.1.detector_description: "updated description" } - match: { analysis_config.detectors.1.detector_index: 1 } @@ -1223,17 +1240,23 @@ { "function": "count", "by_field_name": "country", - "rules": [ + "custom_rules": [ { - "actions": ["filter_results", "skip_sampling"], + "actions": ["skip_result", "skip_model_update"], + "scope": { + "country": {"filter_id": "safe_countries"} + }, "conditions": [ { - "type":"numerical_actual", - "field_name":"country", - "field_value": "uk", - "condition": {"operator":"lt","value":"33.3"} + "applies_to":"actual", + "operator":"lt", + "value": 33.3 }, - {"type":"categorical", "field_name":"country", "filter_id": "foo"} + { + "applies_to":"typical", + "operator":"lte", + "value": 42.0 + } ] } ] @@ -1248,83 +1271,23 @@ job_id: jobs-crud-rules - match: { count: 1 } - match: { - jobs.0.analysis_config.detectors.0.rules: [ + jobs.0.analysis_config.detectors.0.custom_rules: [ { - "actions": ["filter_results", "skip_sampling"], - "conditions_connective": "or", + "actions": ["skip_result", "skip_model_update"], + "scope": { + "country": {"filter_id": "safe_countries", "filter_type": "include"} + }, "conditions": [ { - "type":"numerical_actual", - "field_name":"country", - "field_value": "uk", - "condition": {"operator":"lt","value":"33.3"} - }, - {"type":"categorical", "field_name":"country", "filter_id": "foo"} - ] - } - ] - } - ---- -"Test job with pre 6.2 rules": - - - skip: - features: "warnings" - reason: certain rule fields were renamed in 6.2.0 - - - do: - warnings: - - Deprecated field [detector_rules] used, expected [rules] instead - - Deprecated field [rule_action] used, expected [actions] instead - - Deprecated field [rule_conditions] used, expected [conditions] instead - - Deprecated field [condition_type] used, expected [type] instead - - Deprecated field [value_filter] used, expected [filter_id] instead - xpack.ml.put_job: - job_id: jobs-crud-pre-6-2-rules - body: > - { - "analysis_config": { - "detectors": [ - { - "function": "count", - "by_field_name": "country", - "detector_rules": [ - { - "rule_action": "filter_results", - "rule_conditions": [ - { - "condition_type":"numerical_actual", - "field_name":"country", - "field_value": "uk", - "condition": {"operator":"lt","value":"33.3"} - }, - {"type":"categorical", "field_name":"country", "value_filter": "foo"} - ] - } - ] - } - ] + "applies_to":"actual", + "operator":"lt", + "value": 33.3 }, - "data_description" : {} - } - - - do: - xpack.ml.get_jobs: - job_id: jobs-crud-pre-6-2-rules - - match: { count: 1 } - - match: { - jobs.0.analysis_config.detectors.0.rules: [ - { - "actions": ["filter_results"], - "conditions_connective": "or", - "conditions": [ { - "type":"numerical_actual", - "field_name":"country", - "field_value": "uk", - "condition": {"operator":"lt","value":"33.3"} - }, - {"type":"categorical", "field_name":"country", "filter_id": "foo"} + "applies_to":"typical", + "operator":"lte", + "value": 42.0 + } ] } ] diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml index 717be0d6b250f..98ef9b32e3d29 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/put_job.yml @@ -188,3 +188,33 @@ setup: ] } +--- +"Unknown Metric": + + - do: + catch: /Unsupported metric \[does_not_exist\]/ + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + xpack.rollup.put_job: + id: foo + body: > + { + "index_pattern": "foo", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "interval": "1h" + } + }, + "metrics": [ + { + "field": "value_field", + "metrics": ["min", "max", "sum", "does_not_exist"] + } + ] + } + + diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index 5926701ad0490..3461c530b4417 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -31,6 +31,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.ESIntegTestCase.ClusterScope; import org.elasticsearch.test.InternalTestCluster; import org.elasticsearch.test.disruption.ServiceDisruptionScheme; @@ -39,7 +40,6 @@ import org.elasticsearch.xpack.core.XPackClient; import org.elasticsearch.xpack.core.XPackSettings; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; import org.elasticsearch.xpack.core.watcher.WatcherState; import org.elasticsearch.xpack.core.watcher.client.WatcherClient; import org.elasticsearch.xpack.core.watcher.execution.ExecutionState; @@ -94,7 +94,7 @@ import static org.hamcrest.core.IsNot.not; @ClusterScope(scope = SUITE, numClientNodes = 0, transportClientRatio = 0, maxNumDataNodes = 3) -public abstract class AbstractWatcherIntegrationTestCase extends XPackIntegTestCase { +public abstract class AbstractWatcherIntegrationTestCase extends ESIntegTestCase { public static final String WATCHER_LANG = Script.DEFAULT_SCRIPT_LANG; diff --git a/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java b/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java index 26a98691f1323..3467316c24f6c 100644 --- a/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java +++ b/x-pack/qa/audit-tests/src/test/java/org/elasticsearch/xpack/security/audit/IndexAuditIT.java @@ -6,7 +6,6 @@ package org.elasticsearch.xpack.security.audit; import com.carrotsearch.hppc.cursors.ObjectCursor; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.http.message.BasicHeader; import org.elasticsearch.action.admin.indices.template.delete.DeleteIndexTemplateResponse; import org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse; @@ -25,9 +24,8 @@ import org.elasticsearch.test.TestCluster; import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; -import org.elasticsearch.xpack.core.test.ObjectCleanerThreadThreadFilter; import org.elasticsearch.xpack.security.audit.index.IndexAuditTrail; +import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import java.io.IOException; import java.net.InetSocketAddress; @@ -40,7 +38,6 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) public class IndexAuditIT extends ESIntegTestCase { private static final String USER = "test_user"; private static final String PASS = "x-pack-test-password"; diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java index 4e0aa9c7e0613..e9ba002779e37 100644 --- a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java +++ b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/AutodetectMemoryLimitIT.java @@ -37,6 +37,7 @@ public void cleanUpTest() { cleanUp(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/ml-cpp/pulls/122") public void testTooManyPartitions() throws Exception { Detector.Builder detector = new Detector.Builder("count", null); detector.setPartitionFieldName("user"); @@ -77,11 +78,12 @@ public void testTooManyPartitions() throws Exception { // Assert we haven't violated the limit too much GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - assertThat(modelSizeStats.getModelBytes(), lessThan(35000000L)); - assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L)); + assertThat(modelSizeStats.getModelBytes(), lessThan(31500000L)); + assertThat(modelSizeStats.getModelBytes(), greaterThan(24000000L)); assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/ml-cpp/pulls/122") public void testTooManyByFields() throws Exception { Detector.Builder detector = new Detector.Builder("count", null); detector.setByFieldName("user"); @@ -122,11 +124,12 @@ public void testTooManyByFields() throws Exception { // Assert we haven't violated the limit too much GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L)); - assertThat(modelSizeStats.getModelBytes(), greaterThan(30000000L)); + assertThat(modelSizeStats.getModelBytes(), lessThan(31500000L)); + assertThat(modelSizeStats.getModelBytes(), greaterThan(25000000L)); assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/ml-cpp/pulls/122") public void testTooManyByAndOverFields() throws Exception { Detector.Builder detector = new Detector.Builder("count", null); detector.setByFieldName("department"); @@ -171,11 +174,12 @@ public void testTooManyByAndOverFields() throws Exception { // Assert we haven't violated the limit too much GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - assertThat(modelSizeStats.getModelBytes(), lessThan(36000000L)); + assertThat(modelSizeStats.getModelBytes(), lessThan(31500000L)); assertThat(modelSizeStats.getModelBytes(), greaterThan(24000000L)); assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/ml-cpp/pulls/122") public void testManyDistinctOverFields() throws Exception { Detector.Builder detector = new Detector.Builder("sum", "value"); detector.setOverFieldName("user"); @@ -221,9 +225,9 @@ public void testManyDistinctOverFields() throws Exception { // Assert we haven't violated the limit too much GetJobsStatsAction.Response.JobStats jobStats = getJobStats(job.getId()).get(0); ModelSizeStats modelSizeStats = jobStats.getModelSizeStats(); - assertThat(modelSizeStats.getModelBytes(), lessThan(90000000L)); - assertThat(modelSizeStats.getModelBytes(), greaterThan(75000000L)); - assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.OK)); + assertThat(modelSizeStats.getModelBytes(), lessThan(116000000L)); + assertThat(modelSizeStats.getModelBytes(), greaterThan(90000000L)); + assertThat(modelSizeStats.getMemoryStatus(), equalTo(ModelSizeStats.MemoryStatus.HARD_LIMIT)); } private static Map createRecord(long timestamp, String user, String department) { diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java index 47ced4a96dde8..aa53d6255cb8e 100644 --- a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java +++ b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/DetectionRulesIT.java @@ -12,7 +12,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.xpack.core.ml.action.GetRecordsAction; import org.elasticsearch.xpack.core.ml.job.config.AnalysisConfig; -import org.elasticsearch.xpack.core.ml.job.config.Condition; import org.elasticsearch.xpack.core.ml.job.config.DataDescription; import org.elasticsearch.xpack.core.ml.job.config.DetectionRule; import org.elasticsearch.xpack.core.ml.job.config.Detector; @@ -21,10 +20,9 @@ import org.elasticsearch.xpack.core.ml.job.config.MlFilter; import org.elasticsearch.xpack.core.ml.job.config.Operator; import org.elasticsearch.xpack.core.ml.job.config.RuleCondition; -import org.elasticsearch.xpack.core.ml.job.config.RuleConditionType; +import org.elasticsearch.xpack.core.ml.job.config.RuleScope; import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.junit.After; -import org.junit.Ignore; import java.io.IOException; import java.util.ArrayList; @@ -35,9 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isOneOf; @@ -48,38 +44,22 @@ public class DetectionRulesIT extends MlNativeAutodetectIntegTestCase { @After - public void cleanUpTest() throws Exception { + public void cleanUpTest() { cleanUp(); } - @AwaitsFix(bugUrl = "this test is muted temporarily until the new rules implementation is merged in") - public void testNumericalRule() throws Exception { - RuleCondition condition1 = RuleCondition.createNumerical( - RuleConditionType.NUMERICAL_ACTUAL, - "by_field", - "by_field_value_1", - new Condition(Operator.LT, "1000")); - RuleCondition condition2 = RuleCondition.createNumerical( - RuleConditionType.NUMERICAL_ACTUAL, - "by_field", - "by_field_value_2", - new Condition(Operator.LT, "500")); - RuleCondition condition3 = RuleCondition.createNumerical( - RuleConditionType.NUMERICAL_ACTUAL, - "by_field", - "by_field_value_3", - new Condition(Operator.LT, "100")); - DetectionRule rule = new DetectionRule.Builder(Arrays.asList(condition1, condition2, condition3)).build(); - - Detector.Builder detector = new Detector.Builder("max", "value"); - detector.setRules(Arrays.asList(rule)); - detector.setByFieldName("by_field"); + public void testCondition() throws Exception { + DetectionRule rule = new DetectionRule.Builder(Arrays.asList( + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.LT, 100.0) + )).build(); - AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder( - Arrays.asList(detector.build())); + Detector.Builder detector = new Detector.Builder("mean", "value"); + detector.setByFieldName("by_field"); + detector.setRules(Arrays.asList(rule)); + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Arrays.asList(detector.build())); analysisConfig.setBucketSpan(TimeValue.timeValueHours(1)); DataDescription.Builder dataDescription = new DataDescription.Builder(); - Job.Builder job = new Job.Builder("detection-rule-numeric-test"); + Job.Builder job = new Job.Builder("detection-rules-it-test-condition"); job.setAnalysisConfig(analysisConfig); job.setDataDescription(dataDescription); @@ -91,12 +71,11 @@ public void testNumericalRule() throws Exception { int totalBuckets = 2 * 24; // each half of the buckets contains one anomaly for each by field value Set anomalousBuckets = new HashSet<>(Arrays.asList(20, 44)); - List byFieldValues = Arrays.asList("by_field_value_1", "by_field_value_2", "by_field_value_3"); + List byFieldValues = Arrays.asList("low", "high"); Map anomalousValues = new HashMap<>(); - anomalousValues.put("by_field_value_1", 800); - anomalousValues.put("by_field_value_2", 400); - anomalousValues.put("by_field_value_3", 400); - int normalValue = 1; + anomalousValues.put("low", 99); + anomalousValues.put("high", 701); + int normalValue = 400; List data = new ArrayList<>(); for (int bucket = 0; bucket < totalBuckets; bucket++) { for (String byFieldValue : byFieldValues) { @@ -115,27 +94,14 @@ public void testNumericalRule() throws Exception { List records = getRecords(job.getId()); assertThat(records.size(), equalTo(1)); - assertThat(records.get(0).getByFieldValue(), equalTo("by_field_value_3")); + assertThat(records.get(0).getByFieldValue(), equalTo("high")); long firstRecordTimestamp = records.get(0).getTimestamp().getTime(); { // Update rules so that the anomalies suppression is inverted - RuleCondition newCondition1 = RuleCondition.createNumerical( - RuleConditionType.NUMERICAL_ACTUAL, - "by_field", - "by_field_value_1", - new Condition(Operator.GT, "1000")); - RuleCondition newCondition2 = RuleCondition.createNumerical( - RuleConditionType.NUMERICAL_ACTUAL, - "by_field", - "by_field_value_2", - new Condition(Operator.GT, "500")); - RuleCondition newCondition3 = RuleCondition.createNumerical( - RuleConditionType.NUMERICAL_ACTUAL, - "by_field", - "by_field_value_3", - new Condition(Operator.GT, "0")); - DetectionRule newRule = new DetectionRule.Builder(Arrays.asList(newCondition1, newCondition2, newCondition3)).build(); + DetectionRule newRule = new DetectionRule.Builder(Arrays.asList( + new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 700.0) + )).build(); JobUpdate.Builder update = new JobUpdate.Builder(job.getId()); update.setDetectorUpdates(Arrays.asList(new JobUpdate.DetectorUpdate(0, null, Arrays.asList(newRule)))); updateJob(job.getId(), update.build()); @@ -149,18 +115,15 @@ public void testNumericalRule() throws Exception { GetRecordsAction.Request recordsAfterFirstHalf = new GetRecordsAction.Request(job.getId()); recordsAfterFirstHalf.setStart(String.valueOf(firstRecordTimestamp + 1)); records = getRecords(recordsAfterFirstHalf); - assertThat(records.size(), equalTo(2)); - Set secondHaldRecordByFieldValues = records.stream().map(AnomalyRecord::getByFieldValue).collect(Collectors.toSet()); - assertThat(secondHaldRecordByFieldValues, contains("by_field_value_1", "by_field_value_2")); + assertThat(records.size(), equalTo(1)); + assertThat(records.get(0).getByFieldValue(), equalTo("low")); } - @AwaitsFix(bugUrl = "this test is muted temporarily until the new rules implementation is merged in") - public void testCategoricalRule() throws Exception { + public void testScope() throws Exception { MlFilter safeIps = new MlFilter("safe_ips", Arrays.asList("111.111.111.111", "222.222.222.222")); assertThat(putMlFilter(safeIps), is(true)); - RuleCondition condition = RuleCondition.createCategorical("ip", safeIps.getId()); - DetectionRule rule = new DetectionRule.Builder(Collections.singletonList(condition)).build(); + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder().include("ip", "safe_ips")).build(); Detector.Builder detector = new Detector.Builder("count", null); detector.setRules(Arrays.asList(rule)); @@ -169,7 +132,7 @@ public void testCategoricalRule() throws Exception { AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build())); analysisConfig.setBucketSpan(TimeValue.timeValueHours(1)); DataDescription.Builder dataDescription = new DataDescription.Builder(); - Job.Builder job = new Job.Builder("detection-rule-categorical-test"); + Job.Builder job = new Job.Builder("detection-rules-it-test-scope"); job.setAnalysisConfig(analysisConfig); job.setDataDescription(dataDescription); @@ -263,6 +226,70 @@ public void testCategoricalRule() throws Exception { closeJob(job.getId()); } + public void testScopeAndCondition() throws IOException { + // We have 2 IPs and they're both safe-listed. + List ips = Arrays.asList("111.111.111.111", "222.222.222.222"); + MlFilter safeIps = new MlFilter("safe_ips", ips); + assertThat(putMlFilter(safeIps), is(true)); + + // Ignore if ip in safe list AND actual < 10. + DetectionRule rule = new DetectionRule.Builder(RuleScope.builder().include("ip", "safe_ips")) + .setConditions(Arrays.asList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.LT, 10.0))) + .build(); + + Detector.Builder detector = new Detector.Builder("count", null); + detector.setRules(Arrays.asList(rule)); + detector.setOverFieldName("ip"); + + AnalysisConfig.Builder analysisConfig = new AnalysisConfig.Builder(Collections.singletonList(detector.build())); + analysisConfig.setBucketSpan(TimeValue.timeValueHours(1)); + DataDescription.Builder dataDescription = new DataDescription.Builder(); + Job.Builder job = new Job.Builder("detection-rules-it-test-scope-and-condition"); + job.setAnalysisConfig(analysisConfig); + job.setDataDescription(dataDescription); + + registerJob(job); + putJob(job); + openJob(job.getId()); + + long timestamp = 1509062400000L; + List data = new ArrayList<>(); + + // First, 20 buckets with a count of 1 for both IPs + for (int bucket = 0; bucket < 20; bucket++) { + for (String ip : ips) { + data.add(createIpRecord(timestamp, ip)); + } + timestamp += TimeValue.timeValueHours(1).getMillis(); + } + + // Now send anomalous count of 9 for 111.111.111.111 + for (int i = 0; i < 9; i++) { + data.add(createIpRecord(timestamp, "111.111.111.111")); + } + + // and 10 for 222.222.222.222 + for (int i = 0; i < 10; i++) { + data.add(createIpRecord(timestamp, "222.222.222.222")); + } + timestamp += TimeValue.timeValueHours(1).getMillis(); + + // Some more normal buckets + for (int bucket = 0; bucket < 3; bucket++) { + for (String ip : ips) { + data.add(createIpRecord(timestamp, ip)); + } + timestamp += TimeValue.timeValueHours(1).getMillis(); + } + + postData(job.getId(), joinBetween(0, data.size(), data)); + closeJob(job.getId()); + + List records = getRecords(job.getId()); + assertThat(records.size(), equalTo(1)); + assertThat(records.get(0).getOverFieldValue(), equalTo("222.222.222.222")); + } + private String createIpRecord(long timestamp, String ip) throws IOException { Map record = new HashMap<>(); record.put("time", timestamp); diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java index a744f3ebb6380..f70efc72506d3 100644 --- a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java +++ b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/MlNativeAutodetectIntegTestCase.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.ml.integration; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.action.admin.cluster.node.tasks.list.ListTasksRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.Client; @@ -35,7 +34,6 @@ import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.SecuritySettingsSourceField; import org.elasticsearch.transport.Netty4Plugin; import org.elasticsearch.xpack.core.LocalStateCompositeXPackPlugin; @@ -86,7 +84,6 @@ import org.elasticsearch.xpack.core.ml.job.results.Result; import org.elasticsearch.xpack.core.security.SecurityField; import org.elasticsearch.xpack.core.security.authc.TokenMetaData; -import org.elasticsearch.xpack.core.test.ObjectCleanerThreadThreadFilter; import java.io.IOException; import java.net.URISyntaxException; @@ -110,7 +107,6 @@ /** * Base class of ML integration tests that use a native autodetect process */ -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) abstract class MlNativeAutodetectIntegTestCase extends ESIntegTestCase { private List jobs = new ArrayList<>(); diff --git a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java index 8410075ff5e27..6703e4ef2365b 100644 --- a/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java +++ b/x-pack/qa/ml-native-tests/src/test/java/org/elasticsearch/xpack/ml/integration/ScheduledEventsIT.java @@ -20,7 +20,6 @@ import org.elasticsearch.xpack.core.ml.job.results.AnomalyRecord; import org.elasticsearch.xpack.core.ml.job.results.Bucket; import org.junit.After; -import org.junit.Ignore; import java.io.IOException; import java.time.Instant; @@ -42,7 +41,6 @@ public void cleanUpTest() { cleanUp(); } - @AwaitsFix(bugUrl = "this test is muted temporarily until the new rules implementation is merged in") public void testScheduledEvents() throws IOException { TimeValue bucketSpan = TimeValue.timeValueMinutes(30); @@ -154,7 +152,6 @@ public void testScheduledEvents() throws IOException { assertThat(records, is(empty())); } - @AwaitsFix(bugUrl = "this test is muted temporarily until the new rules implementation is merged in") public void testScheduledEventWithInterimResults() throws IOException { TimeValue bucketSpan = TimeValue.timeValueMinutes(30); Job.Builder job = createJob("scheduled-events-interim-results", bucketSpan); @@ -196,7 +193,6 @@ public void testScheduledEventWithInterimResults() throws IOException { /** * Test an open job picks up changes to scheduled events/calendars */ - @AwaitsFix(bugUrl = "this test is muted temporarily until the new rules implementation is merged in") public void testOnlineUpdate() throws Exception { TimeValue bucketSpan = TimeValue.timeValueMinutes(30); Job.Builder job = createJob("scheduled-events-online-update", bucketSpan); diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml index 6ea8771c2374e..c1b238422e92e 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/30_ml_jobs_crud.yml @@ -91,9 +91,9 @@ wait_for_status: green --- -"Test get job with rules": +"Test job with pre 6.4 rules": - do: xpack.ml.get_jobs: - job_id: old-cluster-job-with-rules + job_id: job-with-old-rules - match: { count: 1 } diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml index da36da301c1f8..3a3334f6907e9 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/old_cluster/30_ml_jobs_crud.yml @@ -134,15 +134,15 @@ } --- -"Test job with pre 6.2 rules": +"Test job with pre 6.4 rules": - skip: - version: "6.2.0 - " - reason: "Rules fields were renamed on 6.2.0" + version: "6.4.0 - " + reason: "Rules were replaced by custom_rules on 6.4.0" - do: xpack.ml.put_job: - job_id: old-cluster-job-with-rules + job_id: job-with-old-rules body: > { "analysis_config": { @@ -171,36 +171,22 @@ } --- -"Test job with post 6.2 rules": +"Test job with pre 6.4 rules - dummy job 6.4 onwards": - skip: - version: " - 6.1.99" - reason: "Rules fields were renamed on 6.2.0" + version: " - 6.3.99" + reason: "Rules replaced by custom_rules on 6.4.0" - do: xpack.ml.put_job: - job_id: old-cluster-job-with-rules + job_id: job-with-old-rules body: > { "analysis_config": { "detectors": [ { "function": "count", - "by_field_name": "country", - "rules": [ - { - "actions": ["filter_results"], - "conditions": [ - { - "type":"numerical_actual", - "field_name":"country", - "field_value": "uk", - "condition": {"operator":"lt","value":"33.3"} - }, - {"type":"categorical", "field_name":"country", "filter_id": "foo"} - ] - } - ] + "by_field_name": "country" } ] }, diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml index 91d294572894d..6634722fac977 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/upgraded_cluster/30_ml_jobs_crud.yml @@ -100,29 +100,14 @@ setup: - match: { acknowledged: true } --- -"Test get job with rules": +"Test job with pre 6.4 rules": - do: xpack.ml.get_jobs: - job_id: old-cluster-job-with-rules + job_id: job-with-old-rules - match: { count: 1 } - - match: { - jobs.0.analysis_config.detectors.0.rules: [ - { - "actions": ["filter_results"], - "conditions_connective": "or", - "conditions": [ - { - "type":"numerical_actual", - "field_name":"country", - "field_value": "uk", - "condition": {"operator":"lt","value":"33.3"} - }, - {"type":"categorical", "field_name":"country", "filter_id": "foo"} - ] - } - ] - } + - is_false: jobs.0.analysis_config.detectors.0.rules + - is_false: jobs.0.analysis_config.detectors.0.custom_rules --- "Test get job with function shortcut should expand": diff --git a/x-pack/qa/security-client-tests/build.gradle b/x-pack/qa/security-client-tests/build.gradle index 9706d554e95c9..4e517f4d3633e 100644 --- a/x-pack/qa/security-client-tests/build.gradle +++ b/x-pack/qa/security-client-tests/build.gradle @@ -3,7 +3,6 @@ apply plugin: 'elasticsearch.rest-test' dependencies { testCompile project(path: xpackModule('core'), configuration: 'runtime') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') } diff --git a/x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java b/x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java index 5342c2bd78095..519f365d515a0 100644 --- a/x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java +++ b/x-pack/qa/security-client-tests/src/test/java/org/elasticsearch/xpack/security/qa/SecurityTransportClientIT.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.security.qa; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.elasticsearch.ElasticsearchSecurityException; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; import org.elasticsearch.action.admin.cluster.node.info.NodeInfo; @@ -16,11 +15,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.core.XPackClientPlugin; +import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.test.ObjectCleanerThreadThreadFilter; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; import java.util.Collection; import java.util.Collections; @@ -34,9 +32,7 @@ /** * Integration tests that test a transport client with security being loaded that connect to an external cluster */ -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) -public class SecurityTransportClientIT extends XPackIntegTestCase { - +public class SecurityTransportClientIT extends ESIntegTestCase { static final String ADMIN_USER_PW = "test_user:x-pack-test-password"; static final String TRANSPORT_USER_PW = "transport:x-pack-test-password"; diff --git a/x-pack/qa/security-example-spi-extension/build.gradle b/x-pack/qa/security-example-spi-extension/build.gradle index 94a7bccca8034..b2fac075cb315 100644 --- a/x-pack/qa/security-example-spi-extension/build.gradle +++ b/x-pack/qa/security-example-spi-extension/build.gradle @@ -9,7 +9,6 @@ esplugin { dependencies { compileOnly project(path: xpackModule('core'), configuration: 'runtime') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') } diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java index 8f8f45f0448b6..65ec595a0d409 100644 --- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java +++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/realm/CustomRealmIT.java @@ -3,7 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - package org.elasticsearch.example.realm; import org.apache.http.message.BasicHeader; @@ -20,9 +19,9 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.env.Environment; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; import org.elasticsearch.xpack.core.XPackClientPlugin; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; import java.util.Collection; import java.util.Collections; @@ -33,7 +32,7 @@ /** * Integration test to test authentication with the custom realm */ -public class CustomRealmIT extends XPackIntegTestCase { +public class CustomRealmIT extends ESIntegTestCase { @Override protected Settings externalClusterClientSettings() { diff --git a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java index 85b34a9612f46..4e1fb72256086 100644 --- a/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java +++ b/x-pack/qa/security-example-spi-extension/src/test/java/org/elasticsearch/example/role/CustomRolesProviderIT.java @@ -18,7 +18,6 @@ import org.elasticsearch.xpack.core.XPackClientPlugin; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; import java.util.Collection; import java.util.Collections; @@ -32,7 +31,7 @@ /** * Integration test for custom roles providers. */ -public class CustomRolesProviderIT extends XPackIntegTestCase { +public class CustomRolesProviderIT extends ESIntegTestCase { private static final String TEST_USER = "test_user"; private static final String TEST_PWD = "change_me"; diff --git a/x-pack/qa/security-migrate-tests/build.gradle b/x-pack/qa/security-migrate-tests/build.gradle index de6f1c86b993f..7ccf6d2349b8b 100644 --- a/x-pack/qa/security-migrate-tests/build.gradle +++ b/x-pack/qa/security-migrate-tests/build.gradle @@ -3,7 +3,6 @@ apply plugin: 'elasticsearch.rest-test' dependencies { testCompile project(path: xpackModule('core'), configuration: 'runtime') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackModule('security'), configuration: 'runtime') testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') } diff --git a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java index e7f3a5ef48061..2987c1afc8daf 100644 --- a/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java +++ b/x-pack/qa/security-migrate-tests/src/test/java/org/elasticsearch/xpack/security/MigrateToolTestCase.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.security; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -14,10 +13,8 @@ import org.elasticsearch.common.logging.ESLoggerFactory; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.transport.TransportAddress; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.test.ObjectCleanerThreadThreadFilter; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -43,7 +40,6 @@ * then run JUnit. If you changed the default port, set "tests.cluster=localhost:PORT" when running * your test. */ -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") public abstract class MigrateToolTestCase extends LuceneTestCase { diff --git a/x-pack/qa/smoke-test-plugins-ssl/build.gradle b/x-pack/qa/smoke-test-plugins-ssl/build.gradle index 7c1f7a8d0e558..595c562af3707 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/build.gradle +++ b/x-pack/qa/smoke-test-plugins-ssl/build.gradle @@ -16,7 +16,6 @@ apply plugin: 'elasticsearch.rest-test' dependencies { testCompile project(path: xpackModule('core'), configuration: 'runtime') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') } String outputDir = "${buildDir}/generated-resources/${project.name}" diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java index 02c2faad2b085..f8d1dd5e2b717 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java +++ b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java @@ -17,7 +17,6 @@ import org.elasticsearch.xpack.core.action.XPackUsageResponse; import org.elasticsearch.xpack.core.monitoring.MonitoringFeatureSetUsage; import org.elasticsearch.xpack.core.security.SecurityField; -import org.elasticsearch.xpack.core.test.XPackIntegTestCase; import org.junit.After; import org.junit.Before; @@ -42,7 +41,7 @@ * then uses a transport client to check that the data have been correctly received and * indexed in the cluster. */ -public class SmokeTestMonitoringWithSecurityIT extends XPackIntegTestCase { +public class SmokeTestMonitoringWithSecurityIT extends ESIntegTestCase { private static final String USER = "test_user"; private static final String PASS = "x-pack-test-password"; private static final String MONITORING_PATTERN = ".monitoring-*"; diff --git a/x-pack/qa/transport-client-tests/build.gradle b/x-pack/qa/transport-client-tests/build.gradle index d179fee378c63..c864a9084cba8 100644 --- a/x-pack/qa/transport-client-tests/build.gradle +++ b/x-pack/qa/transport-client-tests/build.gradle @@ -3,7 +3,6 @@ apply plugin: 'elasticsearch.rest-test' dependencies { testCompile project(path: xpackModule('core'), configuration: 'runtime') - testCompile project(path: xpackModule('core'), configuration: 'testArtifacts') testCompile project(path: xpackProject('transport-client').path, configuration: 'runtime') } diff --git a/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java b/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java index 4eb845ec9e90e..c77715431ec5e 100644 --- a/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java +++ b/x-pack/qa/transport-client-tests/src/test/java/org/elasticsearch/xpack/ml/client/ESXPackSmokeClientTestCase.java @@ -5,7 +5,6 @@ */ package org.elasticsearch.xpack.ml.client; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; import org.apache.logging.log4j.Logger; import org.apache.lucene.util.LuceneTestCase; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -16,7 +15,6 @@ import org.elasticsearch.common.transport.TransportAddress; import org.elasticsearch.env.Environment; import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient; -import org.elasticsearch.xpack.core.test.ObjectCleanerThreadThreadFilter; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -48,7 +46,6 @@ * test. */ @LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose") -@ThreadLeakFilters(filters = {ObjectCleanerThreadThreadFilter.class}) public abstract class ESXPackSmokeClientTestCase extends LuceneTestCase { /**