From 0c99c5d7a50af50360a4c400082d7693afbeb5aa Mon Sep 17 00:00:00 2001 From: andriikorotkov <88329385+andriikorotkov@users.noreply.github.com> Date: Fri, 17 Dec 2021 12:47:45 +0200 Subject: [PATCH] :tada: Setting CPU and Memory limits for performance tests via github (#8807) * updated ci performance test sh script * updated ci performance test sh script * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * set all params to one arg * run tests * update ci creds * update performance test yaml file * update performance test yaml file * update performance test yaml file * update performance test yaml file * update performance test sh file * update performance test sh file * update performance test sh file * update performance test sh file * update java to 17 version in yaml file for performance tests * update java to 17 version in yaml file for performance tests * update java to 17 version in yaml file for performance tests * update java to 17 version in yaml file for performance tests * update java to 17 version in yaml file for performance tests * clean test logs * remove test logs * remove test logs * remove test logs * return performance test logs * add readme by performance tests --- .../workflows/test-performance-command.yml | 4 ++++ .../source/AbstractSourceConnectorTest.java | 4 ++-- .../connectors/source-mssql/README.md | 11 ++++++++++ .../connectors/source-mysql/README.md | 9 +++++++++ .../connectors/source-postgres/README.md | 10 ++++++++++ tools/bin/ci_performance_test.sh | 20 +++++++++++++------ 6 files changed, 50 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-performance-command.yml b/.github/workflows/test-performance-command.yml index 3db5a501d2466..1cf83a0b24a64 100644 --- a/.github/workflows/test-performance-command.yml +++ b/.github/workflows/test-performance-command.yml @@ -67,6 +67,10 @@ jobs: uses: actions/checkout@v2 with: repository: ${{ github.event.inputs.repo }} + - name: Install Java + uses: actions/setup-java@v1 + with: + java-version: '17' - name: Install Pyenv and Tox # Beside PyEnv, this does not set any runtimes up because it uses an AMI image that has everything pre-installed. See https://github.com/airbytehq/airbyte/issues/4559/ run: | diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java index 760901e4f6446..f4a2f53a068c7 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/AbstractSourceConnectorTest.java @@ -221,8 +221,8 @@ private static Map prepareResourceRequestMapBySystemProperties() if (memoryLimit.isBlank() || memoryLimit.isEmpty()) { memoryLimit = WorkerUtils.DEFAULT_RESOURCE_REQUIREMENTS.getMemoryLimit(); } - LOGGER.error("cpu limit -->> {}", cpuLimit); - LOGGER.error("memory limit -->> {}", memoryLimit); + LOGGER.info("Container CPU Limit = {}", cpuLimit); + LOGGER.info("Container Memory Limit = {}", memoryLimit); Map result = new HashMap<>(); result.put(CPU_REQUEST_FIELD_NAME, WorkerUtils.DEFAULT_RESOURCE_REQUIREMENTS.getCpuRequest()); result.put(CPU_LIMIT_FIELD_NAME, cpuLimit); diff --git a/airbyte-integrations/connectors/source-mssql/README.md b/airbyte-integrations/connectors/source-mssql/README.md index 4b239ea23d197..26bf5d6f386b7 100644 --- a/airbyte-integrations/connectors/source-mssql/README.md +++ b/airbyte-integrations/connectors/source-mssql/README.md @@ -2,6 +2,17 @@ ## Performance Test +### Running performance tests with CPU and Memory limits for the container + +In order to run performance tests with CPU and Memory limits, you need to run the performance test start command with +additional parameters **cpulimit=cpulimit/YOU_CPU_LIMIT** and **memorylimit=memorylimit/YOU_MEMORY_LIMIT**. +**YOU_MEMORY_LIMIT** - RAM limit. Be sure to indicate the limit in MB or GB at the end. Minimum size - 6MB. +**YOU_CPU_LIMIT** - CPU limit. Minimum size - 2. +These parameters are optional and can be used separately from each other. +For example, if you use only **memorylimit=memorylimit/2GB**, only the memory limit for the container will be set, the CPU will not be limited. +Also, if you do not use both of these parameters, then performance tests will run without memory and CPU limitations. + + ### Use MsSQL script to populate the benchmark database In order to create a database with a certain number of tables, and a certain number of records in each of them, diff --git a/airbyte-integrations/connectors/source-mysql/README.md b/airbyte-integrations/connectors/source-mysql/README.md index 1f15ee12662e9..4f968d6c4ba69 100644 --- a/airbyte-integrations/connectors/source-mysql/README.md +++ b/airbyte-integrations/connectors/source-mysql/README.md @@ -37,6 +37,15 @@ To run performance tests: ``` ./gradlew :airbyte-integrations:connectors:source-mysql:performanceTest ``` +### Running performance tests with CPU and Memory limits for the container + +In order to run performance tests with CPU and Memory limits, you need to run the performance test start command with +additional parameters **cpulimit=cpulimit/YOU_CPU_LIMIT** and **memorylimit=memorylimit/YOU_MEMORY_LIMIT**. +**YOU_MEMORY_LIMIT** - RAM limit. Be sure to indicate the limit in MB or GB at the end. Minimum size - 6MB. +**YOU_CPU_LIMIT** - CPU limit. Minimum size - 2. +These parameters are optional and can be used separately from each other. +For example, if you use only **memorylimit=memorylimit/2GB**, only the memory limit for the container will be set, the CPU will not be limited. +Also, if you do not use both of these parameters, then performance tests will run without memory and CPU limitations. ### Use MySQL script to populate the benchmark database diff --git a/airbyte-integrations/connectors/source-postgres/README.md b/airbyte-integrations/connectors/source-postgres/README.md index 87d2d41f7f032..6ba53763dd696 100644 --- a/airbyte-integrations/connectors/source-postgres/README.md +++ b/airbyte-integrations/connectors/source-postgres/README.md @@ -2,6 +2,16 @@ ## Performance Test +### Running performance tests with CPU and Memory limits for the container + +In order to run performance tests with CPU and Memory limits, you need to run the performance test start command with +additional parameters **cpulimit=cpulimit/YOU_CPU_LIMIT** and **memorylimit=memorylimit/YOU_MEMORY_LIMIT**. +**YOU_MEMORY_LIMIT** - RAM limit. Be sure to indicate the limit in MB or GB at the end. Minimum size - 6MB. +**YOU_CPU_LIMIT** - CPU limit. Minimum size - 2. +These parameters are optional and can be used separately from each other. +For example, if you use only **memorylimit=memorylimit/2GB**, only the memory limit for the container will be set, the CPU will not be limited. +Also, if you do not use both of these parameters, then performance tests will run without memory and CPU limitations. + ### Use Postgres script to populate the benchmark database In order to create a database with a certain number of tables, and a certain number of records in each of them, diff --git a/tools/bin/ci_performance_test.sh b/tools/bin/ci_performance_test.sh index 85a879017305b..3caa5154ad40a 100755 --- a/tools/bin/ci_performance_test.sh +++ b/tools/bin/ci_performance_test.sh @@ -7,8 +7,6 @@ set -e # runs performance tests for an performance name connector="$1" -firstarg="" -secondarg="" if [[ "$2" ]]; then if [[ "$2" == *"cpulimit"* ]]; then firstarg="-DcpuLimit=$(echo $2 | cut -d / -f 2)" @@ -46,14 +44,24 @@ else elif [[ "$connector" == *"connectors"* ]]; then connector_name=$(echo $connector | cut -d / -f 2) selected_performance_test=$(echo "$all_performance_tests" | grep "^$connector_name$" || echo "") - performanceTestCommand="$(_to_gradle_path "airbyte-integrations/$connector" performanceTest) $firstarg $secondargt" + performanceTestCommand="$(_to_gradle_path "airbyte-integrations/$connector" performanceTest)" else selected_performance_test=$(echo "$all_performance_tests" | grep "^$connector$" || echo "") - performanceTestCommand=":airbyte-integrations:connectors:$connector:performanceTest $firstarg $secondarg" + performanceTestCommand=":airbyte-integrations:connectors:$connector:performanceTest" fi if [ -n "$selected_performance_test" ] ; then - echo "Running: ./gradlew --no-daemon --scan $performanceTestCommand" - ./gradlew --no-daemon --scan "$performanceTestCommand" + if [[ "$firstarg" ]]; then + if [[ "$secondarg" ]]; then + echo "Running: ./gradlew --no-daemon --scan $performanceTestCommand $firstarg $secondarg" + ./gradlew --no-daemon --scan "$performanceTestCommand" "$firstarg" "$secondarg" + else + echo "Running: ./gradlew --no-daemon --scan $performanceTestCommand $firstarg" + ./gradlew --no-daemon --scan "$performanceTestCommand" "$firstarg" + fi + else + echo "Running: ./gradlew --no-daemon --scan $performanceTestCommand" + ./gradlew --no-daemon --scan "$performanceTestCommand" + fi else echo "Connector '$connector' not found..." return 1