separate {restart,libc} test

.github/actions/prepare_libclient_test/action.yml

@@ -0,0 +1,33 @@
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Docker buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v2
+
+    - name: Build libc with cache
+      uses: ./.github/actions/build_images_with_cache
+      with:
+        container-name: "medium_libc"
+        cache-key: ${{ hashFiles('./packages/client/libclient-py/**') }}
+        dockerfile-path: ./scripts/libclient/Dockerfile
+        builder-name: ${{ steps.buildx.outputs.name }}
+        target: "medium"
+
+    - name: Build cc with cache
+      uses: ./.github/actions/build_images_with_cache
+      with:
+        container-name: "dev_cc"
+        cache-key: ${{ hashFiles('./packages/server/computation_container/**') }}
+        dockerfile-path: ./packages/server/computation_container/Dockerfile
+        builder-name: ${{ steps.buildx.outputs.name }}
+        target: "dev"
+
+    - name: Build mc with cache
+      uses: ./.github/actions/build_images_with_cache
+      with:
+        container-name: "dev_mc"
+        cache-key: ${{ hashFiles('./packages/server/manage_container/**') }}
+        dockerfile-path: ./packages/server/manage_container/Dockerfile
+        builder-name: ${{ steps.buildx.outputs.name }}
+        target: "dev"

.github/workflows/test-server-all.yml

@@ -10,6 +10,7 @@ on:
 env:
   DOCKER_BUILDKIT: 1
   COMPOSE_DOCKER_CLI_BUILD: 1
+  COMPOSE_FILES_OPT: "-f ./docker-compose.medium.yml -f ./docker-compose.core.yml -f ./docker-compose.benchmark.yml -f ./docker-compose.small.yml -f ./docker-compose.datadog.yml -f ./docker-compose.dev.yml"
 
 jobs:
   lint:
@@ -32,7 +33,6 @@ jobs:
 
   small_test_cc:
     runs-on: ubuntu-latest
-    needs: [lint]
 
     steps:
       - name: Checkout
@@ -63,7 +63,6 @@ jobs:
 
   small_test_mc:
     runs-on: ubuntu-latest
-    needs: [lint]
 
     steps:
       - name: Checkout
@@ -88,7 +87,6 @@ jobs:
 
   small_test_bts:
     runs-on: ubuntu-latest
-    needs: [lint]
 
     steps:
       - name: Checkout
@@ -113,15 +111,14 @@ jobs:
 
   small_test:
     runs-on: ubuntu-latest
-    needs: [lint, small_test_cc, small_test_mc, small_test_bts]
+    needs: [small_test_cc, small_test_mc, small_test_bts]
 
     steps:
       - name: Merge All small_test
         run: echo "ok"
 
   medium_test_cc:
     runs-on: ubuntu-latest
-    needs: [lint]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -163,7 +160,6 @@ jobs:
 
   medium_test_mc:
     runs-on: ubuntu-latest
-    needs: [lint]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -187,7 +183,23 @@ jobs:
 
   medium_test_libclient:
     runs-on: ubuntu-latest
-    needs: [lint]
+    strategy:
+      fail-fast: false
+      matrix:
+        test_name:
+          [
+            correl,
+            get_elapsed_time,
+            job_error_info,
+            join,
+            mean,
+            parallel,
+            send_share,
+            string,
+            sum,
+            token,
+            variance
+          ]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -205,44 +217,14 @@ jobs:
       - name: Log into registry
         run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
 
-      - name: Set up Docker buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Build libc with cache
-        uses: ./.github/actions/build_images_with_cache
-        with:
-          container-name: "medium_libc"
-          cache-key: ${{ hashFiles('./packages/client/libclient-py/**') }}
-          dockerfile-path: ./scripts/libclient/Dockerfile
-          builder-name: ${{ steps.buildx.outputs.name }}
-          target: "medium"
-
-      - name: Build cc with cache
-        uses: ./.github/actions/build_images_with_cache
-        with:
-          container-name: "dev_cc"
-          cache-key: ${{ hashFiles('./packages/server/computation_container/**') }}
-          dockerfile-path: ./packages/server/computation_container/Dockerfile
-          builder-name: ${{ steps.buildx.outputs.name }}
-          target: "dev"
-
-      - name: Build mc with cache
-        uses: ./.github/actions/build_images_with_cache
-        with:
-          container-name: "dev_mc"
-          cache-key: ${{ hashFiles('./packages/server/manage_container/**') }}
-          dockerfile-path: ./packages/server/manage_container/Dockerfile
-          builder-name: ${{ steps.buildx.outputs.name }}
-          target: "dev"
+      - uses: ./.github/actions/prepare_libclient_test
 
-      - name: Run bazel test in libclient
-        run: make test t=./libclient m=run p=medium
+      - name: Run container test
+        run: docker-compose ${{ env.COMPOSE_FILES_OPT }} run medium-libc pytest src/tests/test_${{ matrix.test_name }}.py -s -v -log-cli-level=DEBUG
         working-directory: ./scripts
 
   medium_test_container_up:
     runs-on: ubuntu-latest
-    needs: [lint]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -266,12 +248,22 @@ jobs:
         run: |
           pipenv sync
           pipenv install --skip-lock ../../packages/client/libclient-py
-          pipenv run pytest ./tests/test_up.py -s -v -log-cli-level=DEBUG --durations=0
+          pipenv run pytest ./tests/test_up.py -s -v -log-cli-level=DEBUG
         working-directory: ./scripts/container_test
 
   medium_test_container_restart:
     runs-on: ubuntu-latest
-    needs: [lint]
+    strategy:
+      fail-fast: false
+      matrix:
+        test_name:
+          [
+            send_share,
+            execute_computation,
+            execute_computations,
+            get_computation_result,
+            execute_multiple
+          ]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -295,12 +287,20 @@ jobs:
         run: |
           pipenv sync
           pipenv install --skip-lock ../../packages/client/libclient-py
-          pipenv run pytest ./tests/test_restart_request.py -s -v -log-cli-level=DEBUG --durations=0
+          pipenv run pytest ./tests/test_restart_request.py::test_success_${{ matrix.test_name }}_with_restart -s -v -log-cli-level=DEBUG
         working-directory: ./scripts/container_test
 
-  medium_test_container_down_test_failed_send_share_with_down:
+  medium_test_container_down:
     runs-on: ubuntu-latest
-    needs: [lint]
+    strategy:
+      fail-fast: false
+      matrix:
+        test_name:
+          [
+            send_share,
+            execute_computation,
+            get_computation_result,
+          ]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -324,112 +324,30 @@ jobs:
         run: |
           pipenv sync
           pipenv install --skip-lock ../../packages/client/libclient-py
-          pipenv run pytest ./tests/test_down_request.py::test_failed_send_share_with_down -s -v -log-cli-level=DEBUG --durations=0
+          pipenv run pytest ./tests/test_down_request.py::test_failed_${{ matrix.test_name }}_with_down -s -v -log-cli-level=DEBUG
         working-directory: ./scripts/container_test
 
-  medium_test_container_down_test_failed_execute_computation_with_down_0:
+  medium_test_container:
     runs-on: ubuntu-latest
-    needs: [lint]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.CI_REPOSITORY_ACCESS_TOKEN }}
-          submodules: true
-
-      - name: Update .bazelrc for using remote cache
-        run: |
-          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
-        env:
-          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
-
-      - uses: ./.github/actions/prepare_container_test
-
-      - name: Run container test
-        run: |
-          pipenv sync
-          pipenv install --skip-lock ../../packages/client/libclient-py
-          pipenv run pytest ./tests/test_down_request.py::test_failed_execute_computation_with_down -k [down_container0] -s -v -log-cli-level=DEBUG --durations=0
-        working-directory: ./scripts/container_test
-
-  medium_test_container_down_test_failed_execute_computation_with_down_1:
-    runs-on: ubuntu-latest
-    needs: [lint]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.CI_REPOSITORY_ACCESS_TOKEN }}
-          submodules: true
-
-      - name: Update .bazelrc for using remote cache
-        run: |
-          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
-        env:
-          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
-
-      - uses: ./.github/actions/prepare_container_test
-
-      - name: Run container test
-        run: |
-          pipenv sync
-          pipenv install --skip-lock ../../packages/client/libclient-py
-          pipenv run pytest ./tests/test_down_request.py::test_failed_execute_computation_with_down -k [down_container1] -s -v -log-cli-level=DEBUG --durations=0
-        working-directory: ./scripts/container_test
-
-  medium_test_container_down_test_failed_get_computation_result_with_down:
-    runs-on: ubuntu-latest
-    needs: [lint]
+    needs:
+      [
+        medium_test_container_up,
+        medium_test_container_restart,
+        medium_test_container_down
+      ]
     steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-          token: ${{ secrets.CI_REPOSITORY_ACCESS_TOKEN }}
-          submodules: true
-
-      - name: Update .bazelrc for using remote cache
-        run: |
-          eval "echo \"$(cat ./.github/workflows/.bazelrc_for_ci)\"" >> ./packages/server/computation_container/.bazelrc
-        env:
-          BUILDBUDDY_API_KEY: ${{ secrets.BUILDBUDDY_API_KEY }}
-
-      - name: Log into registry
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
-
-      - uses: ./.github/actions/prepare_container_test
-
-      - name: Run container test
-        run: |
-          pipenv sync
-          pipenv install --skip-lock ../../packages/client/libclient-py
-          pipenv run pytest ./tests/test_down_request.py::test_failed_get_computation_result_with_down -s -v -log-cli-level=DEBUG --durations=0
-        working-directory: ./scripts/container_test
+      - name: Merge All medium_test_container
+        run: echo "ok"
 
   medium_test:
     runs-on: ubuntu-latest
     needs:
       [
-        lint,
         medium_test_cc,
         medium_test_mc,
         medium_test_libclient,
-        medium_test_container_up,
-        medium_test_container_restart,
-        medium_test_container_down_test_failed_send_share_with_down,
-        medium_test_container_down_test_failed_execute_computation_with_down_0,
-        medium_test_container_down_test_failed_execute_computation_with_down_1,
-        medium_test_container_down_test_failed_get_computation_result_with_down,
+        medium_test_container
       ]
-
     steps:
       - name: Merge All medium_test
         run: echo "ok"

scripts/container_test/tests/test_restart_request.py

@@ -10,7 +10,7 @@
         (mc_p(1)),
     ]
 )
-def test_succes_send_share_with_restart(restart_container):
+def test_success_send_share_with_restart(restart_container):
     # 特定のコンテナをrestartさせてsend_shareを送れるか
     Containers.down_all()
     mc_all().up()

scripts/docker-compose.medium.yml

@@ -226,5 +226,5 @@ services:
     command:
       - /bin/bash
       - '-c'
-      - pip list && pytest src/tests -s -v -log-cli-level=DEBUG
+      - pip list
     network_mode: host

scripts/libclient/medium_test.sh

@@ -14,7 +14,7 @@ setup() {
 # run処理を記述する関数
 # NOTE: この関数は例外的にワンライナーで書かなくて良い
 run() {
-    docker-compose $COMPOSE_FILES_OPT up medium-libc
+    docker-compose $COMPOSE_FILES_OPT run medium-libc pytest src/tests -s -v -log-cli-level=DEBUG
 }
 
 # runの後に実行されるteardown処理を記述する関数

scripts/libclient/src/utils.py

@@ -33,14 +33,14 @@ def __try_get_computation_result(job_uuid, is_limit):
     return None
 
 def get_result(res, limit=20):
-    """ 計算が終わるまで結果取得をし続ける(5秒x20回) """
+    """ 計算が終わるまで limit 回結果取得をし続ける i回目の終わりにi秒待つ"""
     if not res["is_ok"]:
         return res
 
     job_uuid = res["job_uuid"]
 
     for i in range(limit):
+        time.sleep(i)
         res = __try_get_computation_result(job_uuid, i+1 == limit)
         if res is not None:
             return res
-        time.sleep(5)