Scripting: Deprecate general cache settings (#55753)

Backport: ef543b0

Author: stu-elastic

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -222,7 +222,7 @@ class ClusterFormationTasks {
         if (distro.equals("oss")) {
             snapshotProject = "oss-" + snapshotProject
         }
-        
+
         BwcVersions.UnreleasedVersionInfo unreleasedInfo = null
 
         if (project.hasProperty('bwcVersions')) {
@@ -420,7 +420,11 @@ class ClusterFormationTasks {
             esConfig['cluster.routing.allocation.disk.watermark.flood_stage'] = '1b'
         }
         // increase script compilation limit since tests can rapid-fire script compilations
-        esConfig['script.max_compilations_rate'] = '2048/1m'
+        if (node.nodeVersion.onOrAfter('7.9.0')) {
+            esConfig['script.disable_max_compilations_rate'] = 'true'
+        } else {
+            esConfig['script.max_compilations_rate'] = '2048/1m'
+        }
         // Temporarily disable the real memory usage circuit breaker. It depends on real memory usage which we have no full control
         // over and the REST client will not retry on circuit breaking exceptions yet (see #31986 for details). Once the REST client
         // can retry on circuit breaking exceptions, we can revert again to the default configuration.

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -1093,7 +1093,11 @@ private void createConfiguration() {
         baseConfig.put("cluster.routing.allocation.disk.watermark.low", "1b");
         baseConfig.put("cluster.routing.allocation.disk.watermark.high", "1b");
         // increase script compilation limit since tests can rapid-fire script compilations
-        baseConfig.put("script.max_compilations_rate", "2048/1m");
+        if (getVersion().onOrAfter(Version.fromString("7.9.0"))) {
+            baseConfig.put("script.disable_max_compilations_rate", "true");
+        } else {
+            baseConfig.put("script.max_compilations_rate", "2048/1m");
+        }
         if (getVersion().getMajor() >= 6) {
             baseConfig.put("cluster.routing.allocation.disk.watermark.flood_stage", "1b");
         }

distribution/docker/docker-compose.yml

@@ -15,7 +15,6 @@ services:
        - cluster.routing.allocation.disk.watermark.low=1b
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
-       - script.max_compilations_rate=2048/1m
        - node.store.allow_mmap=false  
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
@@ -55,7 +54,6 @@ services:
        - cluster.routing.allocation.disk.watermark.low=1b
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
-       - script.max_compilations_rate=2048/1m
        - node.store.allow_mmap=false  
        - xpack.security.enabled=true
        - xpack.security.transport.ssl.enabled=true
@@ -95,7 +93,6 @@ services:
        - cluster.routing.allocation.disk.watermark.low=1b
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
-       - script.max_compilations_rate=2048/1m
        - node.store.allow_mmap=false  
     volumes: 
        - ./build/oss-repo:/tmp/es-repo
@@ -120,7 +117,6 @@ services:
        - cluster.routing.allocation.disk.watermark.low=1b
        - cluster.routing.allocation.disk.watermark.high=1b
        - cluster.routing.allocation.disk.watermark.flood_stage=1b
-       - script.max_compilations_rate=2048/1m
        - node.store.allow_mmap=false  
     volumes: 
        - ./build/oss-repo:/tmp/es-repo

docs/reference/migration/index.asciidoc

@@ -11,6 +11,7 @@ For information about how to upgrade your cluster, see <<setup-upgrade>>.
 
 --
 
+include::migrate_7_9.asciidoc[]
 include::migrate_7_8.asciidoc[]
 include::migrate_7_7.asciidoc[]
 include::migrate_7_6.asciidoc[]

docs/reference/migration/migrate_7_9.asciidoc

@@ -0,0 +1,71 @@
+[[breaking-changes-7.9]]
+== Breaking changes in 7.9
+++++
+<titleabbrev>7.9</titleabbrev>
+++++
+
+This section discusses the changes that you need to be aware of when migrating
+your application to {es} 7.9.
+
+See also <<release-highlights>> and <<es-release-notes>>.
+
+* <<breaking_79_script_cache_changes>>
+
+//NOTE: The notable-breaking-changes tagged regions are re-used in the
+//Installation and Upgrade Guide
+
+//tag::notable-breaking-changes[]
+[discrete]
+[[breaking_79_script_cache_changes]]
+=== Script cache changes
+[[deprecate_general_script_cache_size]]
+.The `script.cache.max_size` setting is deprecated.
+
+[%collapsible]
+====
+*Details* +
+The `script.cache.max_size` setting is deprecated. In {es} 8.0.0, this is
+set per-context.
+
+*Impact* +
+To avoid deprecation warnings, discontinue use of the `script.cache.max_size`
+setting. You may use `script.context.$CONTEXT.context_max_size` for the particular context.
+For example, for the `ingest` context, use `script.context.ingest.context_max_size`.
+
+====
+
+[discrete]
+[[deprecate_general_script_expire]]
+.The `script.cache.expire` setting is deprecated.
+
+[%collapsible]
+====
+*Details* +
+The `script.cache.expire` setting is deprecated. In {es} 8.0.0, this is
+set per-context.
+
+*Impact* +
+To avoid deprecation warnings, discontinue use of the `script.cache.expire`
+setting. You may use `script.context.$CONTEXT.cache_expire` for the particular context.
+For example, for the `update` context, use `script.context.update.cache_expire`.
+
+====
+
+[discrete]
+[[deprecate_general_script_compile_rate]]
+.The `script.max_compilations_rate` setting is deprecated.
+
+[%collapsible]
+====
+*Details* +
+The `script.max_compilations_rate` setting is deprecated. In {es} 8.0.0, this is
+set per-context.
+
+*Impact* +
+To avoid deprecation warnings, discontinue use of the `script.max_compilations_rate`
+setting. You may use `script.context.$CONTEXT.max_compilations_rate` for the particular
+context.  For example, for the `processor_conditional` context, use
+`script.context.processor_conditional.max_compilations_rate`.
+
+====
+//end::notable-breaking-changes[]

docs/reference/modules/indices/circuit_breaker.asciidoc

@@ -111,8 +111,8 @@ within a period of time.
 See the "prefer-parameters" section of the <<modules-scripting-using,scripting>>
 documentation for more information.
 
-`script.max_compilations_rate`::
+`script.context.$CONTEXT.max_compilations_rate`::
 
     Limit for the number of unique dynamic scripts within a certain interval
-    that are allowed to be compiled. Defaults to 75/5m, meaning 75 every 5 
-    minutes.
+    that are allowed to be compiled for a given context. Defaults to `75/5m`,
+    meaning 75 every 5 minutes.

docs/reference/release-notes.asciidoc

@@ -6,6 +6,7 @@
 
 This section summarizes the changes in each release.
 
+* <<release-notes-7.9.0>>
 * <<release-notes-7.8.0>>
 * <<release-notes-7.7.1>>
 * <<release-notes-7.7.0>>
@@ -34,6 +35,7 @@ This section summarizes the changes in each release.
 
 --
 
+include::release-notes/7.9.asciidoc[]
 include::release-notes/7.8.asciidoc[]
 include::release-notes/7.7.asciidoc[]
 include::release-notes/7.6.asciidoc[]

docs/reference/release-notes/7.9.asciidoc

@@ -0,0 +1,11 @@
+[[release-notes-7.9.0]]
+== {es} version 7.9.0
+
+Also see <<breaking-changes-7.9,Breaking changes in 7.9>>.
+
+[[breaking-7.9.0]]
+[float]
+=== Breaking changes
+
+Script Cache::
+* Script cache size and rate limiting are per-context {pull}55753[#55753] (issue: {issue}50152[#50152])

docs/reference/release-notes/highlights.asciidoc

@@ -3,16 +3,16 @@
 
 coming[{minor-version}]
 
-Here are the highlights of what's new and improved in {es} {minor-version}! 
+Here are the highlights of what's new and improved in {es} {minor-version}!
 ifeval::["{release-state}"!="unreleased"]
-For detailed information about this release, see the 
-<<release-notes-{elasticsearch_version}, Release notes >>  and 
+For detailed information about this release, see the
+<<release-notes-{elasticsearch_version}, Release notes >> and
 <<breaking-changes-{minor-version}, Breaking changes>>.
 endif::[]
 
 // Add previous release to the list
-Other versions: 
-{ref-bare}/7.8/release-highlights.html[7.8] 
+Other versions:
+{ref-bare}/7.8/release-highlights.html[7.8]
 | {ref-bare}/7.7/release-highlights.html[7.7]
 | {ref-bare}/7.6/release-highlights-7.6.0.html[7.6]
 | {ref-bare}/7.5/release-highlights-7.5.0.html[7.5]
@@ -24,17 +24,17 @@ Other versions:
 
 
 
-// Use the notable-highlights tag to mark entries that 
+// Use the notable-highlights tag to mark entries that
 // should be featured in the Stack Installation and Upgrade Guide:
-// tag::notable-highlights[] 
+// tag::notable-highlights[]
 // [discrete]
 // === Heading
 //
-// Description. 
+// Description.
 // end::notable-highlights[]
 
 // Omit the notable highlights tag for entries that only need to appear in the ES ref:
-// [float] 
+// [float]
 // === Heading
 //
-// Description. 
+// Description.

docs/reference/scripting/using.asciidoc

@@ -98,9 +98,11 @@ second version is only compiled once.
 
 If you compile too many unique scripts within a small amount of time,
 Elasticsearch will reject the new dynamic scripts with a
-`circuit_breaking_exception` error. By default, up to 15 inline scripts per
-minute will be compiled. You can change this setting dynamically by setting
-`script.max_compilations_rate`.
+`circuit_breaking_exception` error. By default, up to 75 scripts per
+5 minutes will be compiled for most contexts and 375 scripts per 5 minutes
+for `ingest` contexts. You can change these settings dynamically by setting
+`script.context.$CONTEXT.max_compilations_rate` e.g.,
+`script.context.field.max_compilations_rate=100/10m`.
 
 ========================================
 
@@ -135,7 +137,7 @@ The same script in the normal form:
 Scripts may be stored in and retrieved from the cluster state using the
 `_scripts` end-point.
 
-If the {es} {security-features} are enabled, you must have the following 
+If the {es} {security-features} are enabled, you must have the following
 privileges to create, retrieve, and delete stored scripts:
 
 * cluster: `all` or `manage`
@@ -241,9 +243,17 @@ templating language]. See <<search-template>> for more information and examples.
 
 All scripts are cached by default so that they only need to be recompiled
 when updates occur. By default, scripts do not have a time-based expiration, but
-you can change this behavior by using the `script.cache.expire` setting.
-You can configure the size of this cache by using the `script.cache.max_size` setting.
-By default, the cache size is `100`.
+you can configure the size of this cache using the
+`script.context.$CONTEXT.cache_expire` setting.
+By default, the cache size is `100` for all contexts except the `ingest` and the
+`processor_conditional` context, where it is `200`.
+
+|====
+| Context | Default Cache Size
+| `ingest`  | 200
+| `processor_conditional` | 200
+| default | 100
+|====
 
 NOTE: The size of scripts is limited to 65,535 bytes. This can be
 changed by setting `script.max_size_in_bytes` setting to increase that soft