Ultimate test coverage reports (phoenixframework#3110)

* merge test coverages * update deprecated actions, fix artifact paths + update playwright to 1.41.2 * allow coverage >= 85% * fix merge command * update README * use mix test * apply suggestions from review * add default jest reporter back * update playwright to 1.43.1 * fix port in playwright teardown script * use liveview esm script for correct coverage reporting * import isUsedInput from module
SteffenDE · May 3, 2024 · 7218d2e · 7218d2e
1 parent 425537d
commit 7218d2e
Show file tree

Hide file tree

Showing 28 changed files with 1,257 additions and 76 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,6 +20,7 @@ jobs:
             otp: 25.3
             lint: lint
 
+          # update coverage report as well
           - elixir: 1.15.5
             otp: 26.0
             lint: lint
@@ -32,7 +33,7 @@ jobs:
         sudo apt update
         sudo apt install -y inotify-tools
     - name: Checkout
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Set up Elixir
       uses: erlef/setup-beam@v1
@@ -41,7 +42,7 @@ jobs:
         otp-version: ${{ matrix.otp }}
 
     - name: Restore deps and _build cache
-      uses: actions/cache@v2
+      uses: actions/cache@v4
       with:
         path: |
           deps
@@ -73,7 +74,14 @@ jobs:
         MIX_ENV: test
 
     - name: Run tests
-      run: mix test
+      run: mix test --cover --export-coverage default
+
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: mix-test-coverage-${{ matrix.otp }}-${{ matrix.elixir }}
+        path: cover/default.coverdata
+        retention-days: 7
 
   npm_test:
     name: npm test
@@ -87,7 +95,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Elixir
         uses: erlef/setup-beam@v1
@@ -96,7 +104,7 @@ jobs:
           otp-version: ${{ matrix.otp }}
 
       - name: Restore deps and _build cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             deps
@@ -109,42 +117,46 @@ jobs:
         run: mix deps.get --only test
 
       - name: Set up Node.js 18.x
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4
         with:
           node-version: 18.x
 
       - name: Restore npm cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-node-
 
       - name: npm install and test
-        run: |
-          cd assets
-          npm install
-          npm test
+        run: npm run js:test
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: js-unit-coverage
+          path: assets/coverage/
+          retention-days: 7
 
   e2e_test:
     name: e2e test
 
     strategy:
       matrix:
         include:
-          - elixir: 1.16.0
+          - elixir: 1.16.1
             otp: 26.2
 
     runs-on: ubuntu-latest
     container:
-      image: mcr.microsoft.com/playwright:v1.41.0-jammy
+      image: mcr.microsoft.com/playwright:v1.43.1-jammy
     env:
       ImageOS: ubuntu22
       HOME: /root
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: install unzip
         run: apt update && apt -y install unzip
@@ -156,7 +168,7 @@ jobs:
           otp-version: ${{ matrix.otp }}
 
       - name: Restore deps and _build cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: |
             deps
@@ -169,7 +181,7 @@ jobs:
         run: mix deps.get
 
       - name: Restore npm cache
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -187,9 +199,92 @@ jobs:
           npm install
           npm run e2e:test
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: playwright-report
           path: playwright-report/
           retention-days: 7
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: e2e-test-results
+          path: test/e2e/test-results/
+          retention-days: 7
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: mix-e2e-coverage
+          path: cover/e2e.coverdata
+          retention-days: 7
+
+  coverage_report:
+    name: coverage report
+
+    runs-on: ubuntu-latest
+    needs: [mix_test, npm_test, e2e_test]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Elixir
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: 1.15.5
+          otp-version: 26.0
+
+      - name: Download mix unit coverage
+        uses: actions/download-artifact@v4
+        with:
+          # This needs to be updated when changing the test matrix
+          name: mix-test-coverage-26-1.15.5
+          path: cover/
+
+      - name: Download mix e2e coverage
+        uses: actions/download-artifact@v4
+        with:
+          name: mix-e2e-coverage
+          path: cover/
+
+      - name: Restore deps and _build cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            deps
+            _build
+          key: deps-${{ runner.os }}-26-1.15.5-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            deps-${{ runner.os }}-26-1.15.5
+
+      - name: Generate mix coverage report
+        run: mix test.coverage
+
+      - name: Download js-unit-coverage artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: js-unit-coverage
+          path: assets/coverage/
+
+      - name: Download e2e-test-results artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: e2e-test-results
+          path: test/e2e/test-results/
+
+      - name: Set up Node.js 18.x
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18.x
+
+      - name: Merge coverage reports
+        run: npm install && npm run cover:merge
+
+      - name: Upload coverage report
+        uses: actions/upload-artifact@v4
+        with:
+          name: overall-coverage
+          path: cover/
+          retention-days: 7
diff --git a/README.md b/README.md
@@ -166,7 +166,12 @@ $ mix deps.get
 $ mix test
 ```
 
-Running the Javascript tests:
+Running all JavaScript tests:
+```bash
+$ npm run test
+```
+
+Running the JavaScript unit tests:
 
 ```bash
 $ cd assets
@@ -176,4 +181,23 @@ $ npm run test
 $ npm run test.watch
 ```
 
+or simply:
+
+```bash
+$ npm run js:test
+```
+
+Running the JavaScript end-to-end tests:
+
+```bash
+$ npm run e2e:test
+```
+
+Checking test coverage:
+
+```bash
+$ npm run cover
+$ npm run cover:report
+```
+
 JS contributions are very welcome, but please do not include an updated `priv/static/phoenix_live_view.js` in pull requests. The maintainers will update it as part of the release process.
diff --git a/assets/jest.config.js b/assets/jest.config.js
@@ -26,20 +26,18 @@ module.exports = {
   // coverageDirectory: undefined,
 
   // An array of regexp pattern strings used to skip coverage collection
-  // coveragePathIgnorePatterns: [
-  //   "/node_modules/"
-  // ],
+  coveragePathIgnorePatterns: [
+    "/node_modules/",
+    "/test/"
+  ],
+
+  collectCoverage: true,
 
   // Indicates which provider should be used to instrument code for coverage
   coverageProvider: "v8",
 
   // A list of reporter names that Jest uses when writing coverage reports
-  // coverageReporters: [
-  //   "json",
-  //   "text",
-  //   "lcov",
-  //   "clover"
-  // ],
+  coverageReporters: ["none"],
 
   // An object that configures minimum threshold enforcement for coverage results
   // coverageThreshold: undefined,
@@ -104,7 +102,18 @@ module.exports = {
   // projects: undefined,
 
   // Use this configuration option to add custom reporters to Jest
-  // reporters: undefined,
+  reporters: [
+    "default",
+    ["jest-monocart-coverage", {
+      name: "Phoenix LiveView JS Unit Coverage",
+      reports: [
+        ["raw", {outputDir: "./raw"}],
+        ["v8"],
+        ["console-summary"]
+      ],
+      outputDir: "./coverage"
+    }]
+  ],
 
   // Automatically reset mock state between every test
   // resetMocks: false,