From 1977b77a914ebcf104e0289d5f7e7a9f66a3daa3 Mon Sep 17 00:00:00 2001 From: Ben Hale Date: Thu, 3 May 2018 11:06:57 -0400 Subject: [PATCH] Memory Calculation Headroom This change adds a configuration option that allocates empty overhead in the memory calculation. This configuration defaults to zero, but would be used in cases where the memory calculator cannot accurately allocate memory regions (e.g. 10G heap sizes). [resolves #583] --- config/open_jdk_jre.yml | 1 + config/oracle_jre.yml | 1 + config/sap_machine_jre.yml | 3 ++- config/zulu_jre.yml | 1 + docs/jre-open_jdk_jre.md | 22 ++++++++------- docs/jre-oracle_jre.md | 22 ++++++++------- docs/jre-sap_machine_jre.md | 22 ++++++++------- docs/jre-zulu_jre.md | 22 ++++++++------- .../jre/open_jdk_like_memory_calculator.rb | 10 ++++++- .../open_jdk_like_memory_calculator_spec.rb | 27 ++++++++++++++++--- spec/java_buildpack/jre/open_jdk_like_spec.rb | 4 +-- 11 files changed, 91 insertions(+), 44 deletions(-) diff --git a/config/open_jdk_jre.yml b/config/open_jdk_jre.yml index 963b4d02a4..dd37c71575 100644 --- a/config/open_jdk_jre.yml +++ b/config/open_jdk_jre.yml @@ -26,4 +26,5 @@ memory_calculator: version: 3.+ repository_root: "{default.repository.root}/memory-calculator/{platform}/{architecture}" class_count: + headroom: stack_threads: 250 diff --git a/config/oracle_jre.yml b/config/oracle_jre.yml index da9d5e54fe..e8250bb728 100644 --- a/config/oracle_jre.yml +++ b/config/oracle_jre.yml @@ -29,4 +29,5 @@ memory_calculator: version: 3.+ repository_root: "{default.repository.root}/memory-calculator/{platform}/{architecture}" class_count: + headroom: stack_threads: 250 diff --git a/config/sap_machine_jre.yml b/config/sap_machine_jre.yml index e583833d0e..b76c74944c 100644 --- a/config/sap_machine_jre.yml +++ b/config/sap_machine_jre.yml @@ -25,5 +25,6 @@ jvmkill_agent: memory_calculator: version: 3.+ repository_root: "{default.repository.root}/memory-calculator/{platform}/{architecture}" - class_count: + class_count: + headroom: stack_threads: 250 diff --git a/config/zulu_jre.yml b/config/zulu_jre.yml index 695818d102..abd9e8f78a 100755 --- a/config/zulu_jre.yml +++ b/config/zulu_jre.yml @@ -29,4 +29,5 @@ memory_calculator: version: 3.+ repository_root: "{default.repository.root}/memory-calculator/{platform}/{architecture}" class_count: + headroom: stack_threads: 250 diff --git a/docs/jre-open_jdk_jre.md b/docs/jre-open_jdk_jre.md index 7ec65b96b5..0cc5c6867a 100644 --- a/docs/jre-open_jdk_jre.md +++ b/docs/jre-open_jdk_jre.md @@ -91,6 +91,19 @@ The user can change the container's total memory available to influence the JRE Unless the user specifies the heap size Java option (`-Xmx`), increasing or decreasing the total memory available results in the heap size setting increasing or decreasing by a corresponding amount. +#### Loaded Classes + +The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. +If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: + +```yaml +class_count: 500 +``` + +#### Headroom + +A percentage of the total memory allocated to the container to be left as headroom and excluded from the memory calculation. + #### Stack Threads The amount of memory that should be allocated to stacks is given as an amount of memory per @@ -102,15 +115,6 @@ the following example: stack_threads: 500 ``` -#### Loaded Classes - -The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. -If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: - -```yaml -class_count: 500 -``` - #### Java Options If the JRE memory settings need to be fine-tuned, the user can set one or more Java memory options to diff --git a/docs/jre-oracle_jre.md b/docs/jre-oracle_jre.md index 3f4d376d53..4062f38461 100644 --- a/docs/jre-oracle_jre.md +++ b/docs/jre-oracle_jre.md @@ -107,6 +107,19 @@ The user can change the container's total memory available to influence the JRE Unless the user specifies the heap size Java option (`-Xmx`), increasing or decreasing the total memory available results in the heap size setting increasing or decreasing by a corresponding amount. +#### Loaded Classes + +The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. +If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: + +```yaml +class_count: 500 +``` + +#### Headroom + +A percentage of the total memory allocated to the container to be left as headroom and excluded from the memory calculation. + #### Stack Threads The amount of memory that should be allocated to stacks is given as an amount of memory per @@ -118,15 +131,6 @@ the following example: stack_threads: 500 ``` -#### Loaded Classes - -The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. -If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: - -```yaml -class_count: 500 -``` - #### Java Options If the JRE memory settings need to be fine-tuned, the user can set one or more Java memory options to diff --git a/docs/jre-sap_machine_jre.md b/docs/jre-sap_machine_jre.md index 6f063b8698..9e1d4ad057 100644 --- a/docs/jre-sap_machine_jre.md +++ b/docs/jre-sap_machine_jre.md @@ -94,6 +94,19 @@ The user can change the container's total memory available to influence the JRE Unless the user specifies the heap size Java option (`-Xmx`), increasing or decreasing the total memory available results in the heap size setting increasing or decreasing by a corresponding amount. +#### Loaded Classes + +The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. +If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: + +```yaml +class_count: 500 +``` + +#### Headroom + +A percentage of the total memory allocated to the container to be left as headroom and excluded from the memory calculation. + #### Stack Threads The amount of memory that should be allocated to stacks is given as an amount of memory per @@ -105,15 +118,6 @@ the following example: stack_threads: 500 ``` -#### Loaded Classes - -The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. -If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: - -```yaml -class_count: 500 -``` - #### Java Options If the JRE memory settings need to be fine-tuned, the user can set one or more Java memory options to diff --git a/docs/jre-zulu_jre.md b/docs/jre-zulu_jre.md index d8ba2df866..d21ceeb5bd 100755 --- a/docs/jre-zulu_jre.md +++ b/docs/jre-zulu_jre.md @@ -98,6 +98,19 @@ The user can change the container's total memory available to influence the JRE Unless the user specifies the heap size Java option (`-Xmx`), increasing or decreasing the total memory available results in the heap size setting increasing or decreasing by a corresponding amount. +#### Loaded Classes + +The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. +If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: + +```yaml +class_count: 500 +``` + +#### Headroom + +A percentage of the total memory allocated to the container to be left as headroom and excluded from the memory calculation. + #### Stack Threads The amount of memory that should be allocated to stacks is given as an amount of memory per @@ -109,15 +122,6 @@ the following example: stack_threads: 500 ``` -#### Loaded Classes - -The amount of memory that is allocated to metaspace and compressed class space (or, on Java 7, the permanent generation) is calculated from an estimate of the number of classes that will be loaded. The default behaviour is to estimate the number of loaded classes as a fraction of the number of class files in the application. -If a specific number of loaded classes should be used for calculations, then it should be specified as in the following example: - -```yaml -class_count: 500 -``` - #### Java Options If the JRE memory settings need to be fine-tuned, the user can set one or more Java memory options to diff --git a/lib/java_buildpack/jre/open_jdk_like_memory_calculator.rb b/lib/java_buildpack/jre/open_jdk_like_memory_calculator.rb index e424e36b95..9d7618e556 100644 --- a/lib/java_buildpack/jre/open_jdk_like_memory_calculator.rb +++ b/lib/java_buildpack/jre/open_jdk_like_memory_calculator.rb @@ -89,6 +89,10 @@ def class_count(configuration) configuration['class_count'] || (0.35 * actual_class_count(root)).ceil end + def headroom(configuration) + configuration['headroom'] + end + def memory_calculator @droplet.sandbox + "bin/java-buildpack-memory-calculator-#{@version}" end @@ -101,9 +105,13 @@ def memory_calculator_tar def memory_calculation_string(relative_path) memory_calculation_string = [qualify_path(memory_calculator, relative_path)] memory_calculation_string << '-totMemory=$MEMORY_LIMIT' - memory_calculation_string << "-stackThreads=#{stack_threads @configuration}" + + headroom = headroom(@configuration) + memory_calculation_string << "-headRoom=#{headroom}" if headroom + memory_calculation_string << "-loadedClasses=#{class_count @configuration}" memory_calculation_string << "-poolType=#{pool_type}" + memory_calculation_string << "-stackThreads=#{stack_threads @configuration}" memory_calculation_string << '-vmOptions="$JAVA_OPTS"' memory_calculation_string.join(' ') diff --git a/spec/java_buildpack/jre/open_jdk_like_memory_calculator_spec.rb b/spec/java_buildpack/jre/open_jdk_like_memory_calculator_spec.rb index 86d0bd9b68..6deb03b4dd 100644 --- a/spec/java_buildpack/jre/open_jdk_like_memory_calculator_spec.rb +++ b/spec/java_buildpack/jre/open_jdk_like_memory_calculator_spec.rb @@ -91,8 +91,8 @@ command = component.memory_calculation_command expect(command).to eq('CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_like_memory_calculator/bin/' \ - 'java-buildpack-memory-calculator-0.0.0 -totMemory=$MEMORY_LIMIT -stackThreads=200 ' \ - '-loadedClasses=2 -poolType=metaspace -vmOptions="$JAVA_OPTS") && echo JVM Memory ' \ + 'java-buildpack-memory-calculator-0.0.0 -totMemory=$MEMORY_LIMIT -loadedClasses=2 ' \ + '-poolType=metaspace -stackThreads=200 -vmOptions="$JAVA_OPTS") && echo JVM Memory ' \ 'Configuration: $CALCULATED_MEMORY && JAVA_OPTS="$JAVA_OPTS $CALCULATED_MEMORY"') end @@ -111,6 +111,25 @@ expect(environment_variables).to include('MALLOC_ARENA_MAX=2') end + context 'with headroom' do + + let(:configuration) { { 'headroom' => '11', 'stack_threads' => '200' } } + + it 'creates memory calculation command with headroom', + app_fixture: 'jre_memory_calculator_application' do + + java_home.version = version_8 + + command = component.memory_calculation_command + + expect(command).to eq('CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_like_memory_calculator/bin/' \ + 'java-buildpack-memory-calculator-0.0.0 -totMemory=$MEMORY_LIMIT -headRoom=11 ' \ + '-loadedClasses=2 -poolType=metaspace -stackThreads=200 -vmOptions="$JAVA_OPTS") && echo ' \ + 'JVM Memory Configuration: $CALCULATED_MEMORY && JAVA_OPTS="$JAVA_OPTS $CALCULATED_MEMORY"') + end + + end + context 'when java 9' do it 'creates memory calculation command', @@ -121,8 +140,8 @@ command = component.memory_calculation_command expect(command).to eq('CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_like_memory_calculator/bin/' \ - 'java-buildpack-memory-calculator-0.0.0 -totMemory=$MEMORY_LIMIT -stackThreads=200 ' \ - '-loadedClasses=14777 -poolType=metaspace -vmOptions="$JAVA_OPTS") && echo JVM Memory ' \ + 'java-buildpack-memory-calculator-0.0.0 -totMemory=$MEMORY_LIMIT -loadedClasses=14777 ' \ + '-poolType=metaspace -stackThreads=200 -vmOptions="$JAVA_OPTS") && echo JVM Memory ' \ 'Configuration: $CALCULATED_MEMORY && JAVA_OPTS="$JAVA_OPTS $CALCULATED_MEMORY"') end diff --git a/spec/java_buildpack/jre/open_jdk_like_spec.rb b/spec/java_buildpack/jre/open_jdk_like_spec.rb index 9a88a1a20c..5d6eed5198 100644 --- a/spec/java_buildpack/jre/open_jdk_like_spec.rb +++ b/spec/java_buildpack/jre/open_jdk_like_spec.rb @@ -70,7 +70,7 @@ java_home.version = version_7 expect(component.command).to eq('CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_like/bin/' \ 'java-buildpack-memory-calculator-0.0.0 -totMemory=$MEMORY_LIMIT' \ - ' -stackThreads=200 -loadedClasses=0 -poolType=permgen -vmOptions="$JAVA_OPTS")' \ + ' -loadedClasses=0 -poolType=permgen -stackThreads=200 -vmOptions="$JAVA_OPTS")' \ ' && echo JVM Memory Configuration: $CALCULATED_MEMORY && ' \ 'JAVA_OPTS="$JAVA_OPTS $CALCULATED_MEMORY"') @@ -80,7 +80,7 @@ java_home.version = version_8 expect(component.command).to eq('CALCULATED_MEMORY=$($PWD/.java-buildpack/open_jdk_like/bin/' \ 'java-buildpack-memory-calculator-0.0.0 -totMemory=$MEMORY_LIMIT' \ - ' -stackThreads=200 -loadedClasses=0 -poolType=metaspace -vmOptions="$JAVA_OPTS")' \ + ' -loadedClasses=0 -poolType=metaspace -stackThreads=200 -vmOptions="$JAVA_OPTS")' \ ' && echo JVM Memory Configuration: $CALCULATED_MEMORY && ' \ 'JAVA_OPTS="$JAVA_OPTS $CALCULATED_MEMORY"')