benchmark.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache license, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the license for the specific language governing permissions and
# limitations under the license.

name: benchmark

on: [ workflow_dispatch ]

jobs:

  build:

    if: github.repository == 'apache/logging-log4j2'

    runs-on: ubuntu-latest

    steps:

      - name: Checkout repository
        uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748   # 3.0.1

      # JDK 11 is needed for the build.
      # Search `maven-toolchains-plugin` usages for details.
      - name: Set up JDK 11
        uses: actions/setup-java@9519cf1382ac8dc61ad461f7f7cb45f033220189   # 3.1.1
        with:
          distribution: temurin
          java-version: 11
          java-package: jdk
          architecture: x64
          cache: maven

      # JDK 8 is needed for the build, and it is the primary bytecode target.
      # Hence, JDK 8 is set up after 11, so that JAVA_HOME used by Maven during build will point to 8.
      - name: Set up JDK 8
        uses: actions/setup-java@9519cf1382ac8dc61ad461f7f7cb45f033220189   # 3.1.1
        with:
          distribution: temurin
          java-version: 8
          java-package: jdk
          architecture: x64
          cache: maven

      - name: Build with Maven
        shell: bash
        run: |
          ./mvnw \
            --show-version --batch-mode --errors --no-transfer-progress \
            --global-toolchains .github/workflows/maven-toolchains.xml \
            -DskipTests=true \
            --projects log4j-perf \
            --also-make \
            package

      - name: Upload built sources
        uses: actions/upload-artifact@6673cd052c4cd6fcf4b4e6e60ea986c889389535    # 3.0.0
        with:
          name: benchmarks.jar
          path: log4j-perf/target/benchmarks.jar

  run:

    needs: build

    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: [ macos-latest, ubuntu-latest, windows-latest ]
        jdk: [ 11, 17 ]
        concurrency: [ 1, 8 ]
        jmhCommand:
          - "-t $CONCURRENCY -f 3 -wi 3 -w 10s -i 4 -r 20s -prof gc -prof perfnorm -rf json -rff results-layout-jtl.json '.*JsonTemplateLayoutBenchmark.*Jtl4EcsLayout'"
          - "-t $CONCURRENCY -f 3 -wi 3 -w 10s -i 4 -r 20s -prof gc -prof perfnorm -rf json -rff results-util-instant-format.json '.*InstantFormatBenchmark.*'"

    steps:

      - name: Checkout repository
        uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748   # 3.0.1

      - name: Download built sources
        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741    # 3.0.0
        with:
          name: benchmarks.jar
          path: log4j-perf/target

      - name: Set up JDK ${{ matrix.jdk }}
        uses: actions/setup-java@9519cf1382ac8dc61ad461f7f7cb45f033220189   # 3.1.1
        with:
          distribution: temurin
          java-version: ${{ matrix.jdk }}
          java-package: jdk
          architecture: x64
          cache: maven

      - name: Run benchmarks
        timeout-minutes: 120
        shell: bash
        run: |
          export CONCURRENCY=${{ matrix.concurrency }}
          java \
            -jar log4j-perf/target/benchmarks.jar \
            ${{ matrix.jmhCommand }}

      - name: Stage benchmark results for commit
        shell: bash
        run: |

          # Determine the artifact version.
          set -x
          ./mvnw \
            --batch-mode --quiet \
            -DforceStdout=true \
            -Dexpression=project.version \
            help:evaluate \
            | tee mvnw-project-version.out
          echo

          # Determine certain file path properties.
          export REVISION=$(<mvnw-project-version.out)
          export BRANCH_NAME=$(echo "$GITHUB_REF" | sed 's/refs\/heads\///g')
          export INSTANT=$(date -u +%Y%m%d)
          export OS_NAME=$(echo "${{ matrix.os }}" | sed 's/-latest//g')
          export RUN_ID=$(printf "%020d" "$GITHUB_RUN_ID")

          # Move and stage the result files.
          for SRC_FILEPATH in results-*.json; do
            export BENCHMARK_NAME=$(basename "$SRC_FILEPATH" | sed 's/^results-//g' | sed 's/\.json$//g')
            export DST_FILEPATH="benchmark/results/$REVISION/$BRANCH_NAME-$INSTANT-$RUN_ID/$BENCHMARK_NAME-O$OS_NAME-J${{ matrix.jdk }}-C${{ matrix.concurrency }}.json"
            mkdir -v -p $(dirname "$DST_FILEPATH")
            mv -v "$SRC_FILEPATH" "$DST_FILEPATH"
            git add "$DST_FILEPATH"
          done

          # Stash changes for commit.
          git stash save
          git clean -f

      - name: Commit benchmark results
        timeout-minutes: 1
        shell: bash
        run: |

          # Checkout the results branch.
          set -x
          git fetch origin gh-pages
          git checkout -B gh-pages origin/gh-pages

          # Stage the stashed changes.
          # (`show + apply` is preferred over `pop`, since the latter fails if file already exists.)
          git stash show -p | git apply
          git add benchmark/results
          git status --porcelain
          test $(git status --porcelain | wc -l) -gt 0 || exit 0

          # Commit changes.
          git config user.name github-actions
          git config user.email github-actions@github.com
          git commit -a -m "Add benchmark results for ${{ runner.os }}, JDK ${{ matrix.jdk }}, and concurrency ${{ matrix.concurrency }}."

          # Push changes in a loop to allow concurrent repository modifications.
          export RETRY=0
          while [ 1 ]; do
            set +e
            git push origin gh-pages 2>&1 | tee git-push.out
            if [ $? -eq 0 ]; then
              exit 0
            else
              set -e
              let RETRY+=1
              echo "retry #$RETRY"
              git pull -r origin gh-pages
            fi
          done

  index:

    runs-on: ubuntu-latest
    needs: run

    steps:

      - name: Checkout repository
        uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748   # 3.0.1
        with:
          ref: gh-pages

      - name: Setup Python 3
        uses: actions/setup-python@98f2ad02fd48d057ee3b4d4f66525b231c3e52b6   # 2.0.2
        with:
          python-version: 3.x

      - name: Index benchmark results
        timeout-minutes: 1
        shell: bash
        run: |

          # Configure the git user.
          git config user.name github-actions
          git config user.email github-actions@github.com

          # Push changes in a loop to allow concurrent repository modifications.
          export RETRY=0
          export INDEX_FILEPATH=benchmark/results/index.json
          while [ 1 ]; do

            # Generate the index file.
            python -c '\
              import json, os, re;\
              filepaths=[re.sub("^benchmark/results/", "", os.path.join(root,filename)) \
                  for (root, dirs, filenames) in os.walk("benchmark/results") \
                      for filename in filenames]; \
              filepaths.remove("index.json"); \
              print(json.dumps(filepaths))' \
            >"$INDEX_FILEPATH"

            # Exit if there are no changes, that is, a concurrent job has indexed all results.
            git diff --exit-code "$INDEX_FILEPATH" && exit 0

            # Commit the index file.
            git add "$INDEX_FILEPATH"
            git commit "$INDEX_FILEPATH" -m "Update benchmark results index."

            # Push the index file.
            set +e
            git push origin gh-pages 2>&1 | tee git-push.out
            if [ $? -eq 0 ]; then
              exit 0
            else
              set -e
              let RETRY+=1
              echo "retry #$RETRY"
              git pull -r origin gh-pages
            fi

          done