Update ci workflows (#318)

* Update CI workflows to include more Python and OpenSearch versions and run faster. Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> * Fix CI workflows. Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> * Add `OPENSEARCH_VERSION` variable. Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> * Set `OPENSEARCH_VERSION` to constant. Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> * CHANGELOG.md Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> * Add 2.6.0 version to test matrix. Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> --------- Signed-off-by: Yury-Fridlyand <yury.fridlyand@improving.com> Co-authored-by: Daniel (dB.) Doubrovkine <dblock@amazon.com>
opensearch-project · Mar 13, 2023 · 409c883 · 409c883
1 parent 06f7dd4
commit 409c883
Show file tree

Hide file tree

Showing 4 changed files with 72 additions and 42 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -8,9 +8,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
       - name: Set up Python 3.7
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install dependencies
@@ -23,9 +23,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
       - name: Set up Python 3.7
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - name: Install build tools
@@ -39,30 +39,30 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['2.7', '3.7', '3.8', '3.9', '3.10', '3.11']
         os: [ubuntu-latest]
         experimental: [false]
         include:
-          - python-version: '2.7'
-            os: ubuntu-20.04
+          - os: 'ubuntu-20.04'
+            python-version: '3.5'
             experimental: false
-          - python-version: '3.5'
-            os: ubuntu-20.04
-            experimental: false
-          - python-version: '3.6'
-            os: ubuntu-20.04
+          - os: 'ubuntu-20.04'
+            python-version: '3.6'
             experimental: false
 
     runs-on: ${{ matrix.os }}
     name: test-${{ matrix.python-version }}
     continue-on-error: ${{ matrix.experimental }}
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v1
+        uses: actions/checkout@v3
       - name: Set Up Python - ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+        env:
+          PIP_NO_PYTHON_VERSION_WARNING: 1
+          PIP_DISABLE_PIP_VERSION_CHECK: 1
       - name: Install Dependencies
         run: |
           python -m pip install -r dev-requirements.txt
@@ -75,7 +75,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: '3.x'
       - name: Install dependencies

diff --git a/.github/workflows/integration-unreleased.yml b/.github/workflows/integration-unreleased.yml
@@ -8,28 +8,66 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        entry:
-          - { opensearch_ref: '1.x' }
-          - { opensearch_ref: '2.0' }
-          - { opensearch_ref: '2.x' }
-          - { opensearch_ref: 'main' }
+        opensearch_ref: [ '1.x', '2.x', '2.0', 'main' ]
+
     steps:
       - name: Checkout OpenSearch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           repository: opensearch-project/opensearch
-          ref: ${{ matrix.entry.opensearch_ref }}
+          ref: ${{ matrix.opensearch_ref }}
           path: opensearch
 
-        # This step builds the docker image tagged as opensearch:test. It will be further used in /ci/run-tests to test against unreleased OpenSearch.
-        # Reference: https://github.com/opensearch-project/OpenSearch/blob/2.0/distribution/docker/build.gradle#L190
+      - name: Get OpenSearch branch top
+        id: get-key
+        working-directory: opensearch
+        run: echo key=`git log -1 --format='%H'` >> $GITHUB_OUTPUT
+
+      - name: Restore cached build
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: opensearch/distribution/archives/linux-tar/build/distributions
+          key: ${{ steps.get-key.outputs.key }}
+
+      - uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin' # See 'Supported distributions' for available options
+          java-version: '11'
+
       - name: Assemble OpenSearch
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        working-directory: opensearch
+        run: ./gradlew :distribution:archives:linux-tar:assemble
+
+      - name: Save cached build
+        if: steps.cache-restore.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v3
+        with:
+          path: opensearch/distribution/archives/linux-tar/build/distributions
+          key: ${{ steps.get-key.outputs.key }}
+
+      - name: Run OpenSearch
+        working-directory: opensearch/distribution/archives/linux-tar/build/distributions
         run: |
-          cd opensearch
-          ./gradlew assemble
+          tar xf opensearch-min-*
+          ./opensearch-*/bin/opensearch &
+          for attempt in {1..20}; do sleep 5; if curl -s localhost:9200; then echo '=====> ready'; break; fi; echo '=====> waiting...'; done
 
       - name: Checkout Python Client
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+        with:
+          path: opensearch-py
 
       - name: Run Integration Test
-        run: "./.ci/run-tests false SNAPSHOT"
+        env:
+          OPENSEARCH_URL: http://localhost:9200
+          TEST_TYPE: server
+          OPENSEARCH_VERSION: SNAPSHOT
+        run: |
+          mkdir -p junit
+          python -m pip install -U --no-cache-dir --disable-pip-version-check pip
+          python -m pip install --no-cache-dir --disable-pip-version-check -r dev-requirements.txt
+          python -m pip install -e .
+          python setup.py test
+        working-directory: opensearch-py
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -9,21 +9,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        secured: ["true", "false"]
-        entry:
-          - { opensearch_version: 1.0.1 }
-          - { opensearch_version: 1.1.0 }
-          - { opensearch_version: 1.2.4 }
-          - { opensearch_version: 1.3.7 }
-          - { opensearch_version: 2.0.1 }
-          - { opensearch_version: 2.1.0 }
-          - { opensearch_version: 2.2.1 }
-          - { opensearch_version: 2.3.0 }
-          - { opensearch_version: 2.4.0 }
-          - { opensearch_version: 2.4.1 }
+        opensearch_version: [ '1.0.1', '1.1.0', '1.2.4', '1.3.7', '2.0.1', '2.1.0', '2.2.1', '2.3.0', '2.4.0', '2.5.0', '2.6.0' ]
+        secured: [ "true", "false" ]
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Integ OpenSearch secured=${{ matrix.secured }} version=${{matrix.entry.opensearch_version}}
-        run: "./.ci/run-tests ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }}"
+        run: "./.ci/run-tests ${{ matrix.secured }} ${{ matrix.entry.opensearch_version }}"
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -9,6 +9,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Fixed
 - Fixed import cycle when importing async helpers ([#311](https://github.com/opensearch-project/opensearch-py/pull/311))
 ### Security
+- Fixed CVE-2022-23491 reported in opensearch-dsl-py ([#295](https://github.com/opensearch-project/opensearch-py/pull/295))
+- Update ci workflows ([#318](https://github.com/opensearch-project/opensearch-py/pull/318))
 
 ## [2.2.0]
 ### Added