Merge pull request #50 from stacks-network/feat/mutation-testing

Add custom CI to run mutation testing completely manually by users from a given team

stacks-core/mutation-testing/check-packages-and-shards/README.md

@@ -4,6 +4,12 @@ Checks whether to run mutants on [stackslib](https://github.com/stacks-network/s
 
 ## Documentation
 
+### Inputs
+
+| Input            | Description                                         | Required | Default |
+| ---------------- | --------------------------------------------------- | -------- | ------- |
+| `ignore_timeout` | Whether to ignore the mutants timeout limit or not. |   true   |    -    |
+
 ### Outputs
 | Output | Description |
 | ------------------------------- | ----------------------------------------------------- |
@@ -14,6 +20,7 @@ Checks whether to run mutants on [stackslib](https://github.com/stacks-network/s
 | `run_small_packages` | True if there are mutants on small packages, false otherwise.
 | `small_packages_with_shards` | True if there are more than 79 mutants on small packages.
 | `run_stacks_signer` | True if there are mutants on `stacks-signer` package.
+| `too_many_mutants` | True if the workflow would timeout because of the amount of mutants.
 
 ## Usage
 

stacks-core/mutation-testing/check-packages-and-shards/action.yml

@@ -4,6 +4,11 @@ branding:
   icon: "check"
   color: "gray-dark"
 
+inputs:
+  ignore_timeout:
+    description: "Whether to ignore the mutants timeout limit or not."
+    required: true
+
 outputs:
   run_stackslib:
     description: "True if there are mutants on `stackslib` package."
@@ -26,6 +31,9 @@ outputs:
   run_stacks_signer:
     description: "True if there are mutants on `stacks-signer` package."
     value: ${{ steps.check_packages_and_shards.outputs.run_stacks_signer }}
+  too_many_mutants:
+    description: "True if the workflow would timeout because of the amount of mutants."
+    value: ${{ steps.check_packages_and_shards.outputs.too_many_mutants }}
 
 runs:
   using: "composite"
@@ -47,7 +55,7 @@ runs:
       id: relative_diff
       shell: bash
       run: |
-        git diff $(git merge-base origin/${{ github.base_ref }} HEAD)..HEAD > git.diff
+        git diff $(git merge-base origin/${{ github.base_ref || 'develop' }} HEAD)..HEAD > git.diff
 
     - name: Update git diff
       id: update_git_diff
@@ -154,6 +162,9 @@ runs:
         number_of_stacks_node_mutants=0
         number_of_small_mutants=0
 
+        stackslib_too_many_mutants=false
+        stacks_node_too_many_mutants=false
+
         # If stackslib.txt file exists, count how many mutants there are
         if [[ -s mutants_by_packages/stackslib.txt ]]; then
           number_of_stackslib_mutants=$(cat mutants_by_packages/stackslib.txt | awk 'END { print NR }' | tr -d '[:space:]')
@@ -180,11 +191,17 @@ runs:
 
         # If there are mutants from stackslib package, check whether to run or not and with or without shards
         if [[ $number_of_stackslib_mutants -ne 0 ]]; then
-          echo "run_stackslib=true" >> "$GITHUB_OUTPUT"
-          if [[ $number_of_stackslib_mutants -gt 7 ]]; then
-            echo "stackslib_with_shards=true" >> "$GITHUB_OUTPUT"
-          else
+          if [[ "${{ inputs.ignore_timeout }}" != "true" && $number_of_stackslib_mutants -gt 72 ]]; then
+            stackslib_too_many_mutants=true
+            echo "run_stackslib=false" >> "$GITHUB_OUTPUT"
             echo "stackslib_with_shards=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_stackslib=true" >> "$GITHUB_OUTPUT"
+            if [[ $number_of_stackslib_mutants -gt 7 ]]; then
+              echo "stackslib_with_shards=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "stackslib_with_shards=false" >> "$GITHUB_OUTPUT"
+            fi
           fi
         else
           echo "run_stackslib=false" >> "$GITHUB_OUTPUT"
@@ -193,11 +210,17 @@ runs:
 
         # If there are mutants from stacks-node package, check whether to run or not and with or without shards
         if [[ $number_of_stacks_node_mutants -ne 0 ]]; then
-          echo "run_stacks_node=true" >> "$GITHUB_OUTPUT"
-          if [[ $number_of_stacks_node_mutants -gt 19 ]]; then
-            echo "stacks_node_with_shards=true" >> "$GITHUB_OUTPUT"
-          else
+          if [[ "${{ inputs.ignore_timeout }}" != "true" && $number_of_stacks_node_mutants -gt 540 ]]; then
+            stacks_node_too_many_mutants=true
+            echo "run_stacks_node=false" >> "$GITHUB_OUTPUT"
             echo "stacks_node_with_shards=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_stacks_node=true" >> "$GITHUB_OUTPUT"
+            if [[ $number_of_stacks_node_mutants -gt 19 ]]; then
+              echo "stacks_node_with_shards=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "stacks_node_with_shards=false" >> "$GITHUB_OUTPUT"
+            fi
           fi
         else
           echo "run_stacks_node=false" >> "$GITHUB_OUTPUT"
@@ -217,4 +240,26 @@ runs:
           echo "small_packages_with_shards=false" >> "$GITHUB_OUTPUT"
         fi
 
+        # Print uncaught (missed/timeout/unviable) mutants to summary
+        if [[ $stacks_node_too_many_mutants == true || $stackslib_too_many_mutants == true ]]; then
+          echo "too_many_mutants=true" >> "$GITHUB_OUTPUT"
+
+          echo "# Packages Exceeding Default CI Runtime" >> "$GITHUB_STEP_SUMMARY"
+          echo "The following packages surpass the default Continuous Integration (CI) runtime limit of 6 hours:" >> "$GITHUB_STEP_SUMMARY"
+
+          if [[ $stackslib_too_many_mutants == true ]]; then
+            echo "- stackslib has $number_of_stackslib_mutants mutants, while the limit is 72." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+          if [[ $stacks_node_too_many_mutants == true ]]; then 
+            echo "- stacks-node has $number_of_stacks_node_mutants mutants, while the limit is 540." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+          echo "" >> "$GITHUB_STEP_SUMMARY"
+          echo "To ensure these packages are processed, it's recommended to tag @admin in the pull request (PR) so the CI can be extended for this/these package(s)." >> "$GITHUB_STEP_SUMMARY"
+          echo "Alternatively, you can run the packages locally by following the instructions in the documentation (link to docs), but it will take more time." >> "$GITHUB_STEP_SUMMARY"
+        else
+          echo "too_many_mutants=false" >> "$GITHUB_OUTPUT"
+        fi
+
         exit 0

stacks-core/mutation-testing/output-pr-mutants/README.md

@@ -15,6 +15,7 @@ Write the mutants tested in the previous jobs of the same workflow to github ste
 | `small_packages` | True if there were mutants running on small packages | true |  |
 | `shards_for_small_packages` | True if mutants on small packages were running using matrix | true |  |
 | `stacks_signer` | True if there were mutants running on stacks-signer package | true |  |
+| `too_many_mutants` | True if the workflow would timeout because of the amount of mutants. | true |  |
 
 ## Usage
 

stacks-core/mutation-testing/output-pr-mutants/action.yml

@@ -26,6 +26,8 @@ inputs:
   stacks_signer:
     description: "True if there were mutants running on stacks-signer package."
     required: true
+  too_many_mutants:
+    description: "True if the workflow would timeout because of the amount of mutants."
 
 runs:
   using: "composite"
@@ -255,4 +257,10 @@ runs:
             exit $exit_code
           ;;
         esac
+
+        if ${{ inputs.too_many_mutants }} == "true"; then
+          echo "Mutants on some of the packages were not run because the workflow would have been timeouted."
+          exit 1
+        fi
+
         exit 0

stacks-core/mutation-testing/pr-differences/action.yml

@@ -33,7 +33,7 @@ runs:
       id: relative_diff
       shell: bash
       run: |
-        git diff $(git merge-base origin/${{ github.base_ref }} HEAD)..HEAD > git.diff
+        git diff $(git merge-base origin/${{ github.base_ref || 'develop' }} HEAD)..HEAD > git.diff
 
     - name: Install cargo-mutants
       id: install_cargo_mutants

team-membership/.eslintignore

@@ -0,0 +1,12 @@
+# Ignore built files
+dist
+build
+
+# Ignore node_modules
+node_modules
+
+# Ignore coverage reports
+coverage
+
+# Ignore configuration files
+*.config.js

team-membership/.eslintrc.json

@@ -0,0 +1,19 @@
+{
+  "env": {
+    "node": true,
+    "es2021": true
+  },
+  "extends": ["eslint:recommended"],
+  "parserOptions": {
+    "ecmaVersion": 2021,
+    "sourceType": "module"
+  },
+  "rules": {
+    "indent": ["error", 2],
+    "linebreak-style": ["error", "unix"],
+    "quotes": ["error", "single"],
+    "semi": ["error", "always"],
+    "no-unused-vars": ["warn"],
+    "no-console": ["warn", { "allow": ["warn", "error"] }]
+  }
+}

team-membership/.gitignore

@@ -0,0 +1 @@
+node_modules/

team-membership/.prettierignore

@@ -0,0 +1,16 @@
+# Ignore artifacts:
+dist
+build
+coverage
+
+# Ignore all HTML files:
+*.html
+
+# Ignore node_modules
+node_modules
+
+# Ignore all markdown files:
+*.md
+
+# Ignore package-lock.json
+package-lock.json

team-membership/.prettierrc

@@ -0,0 +1,8 @@
+{
+    "semi": true,
+    "trailingComma": "es5",
+    "singleQuote": true,
+    "printWidth": 100,
+    "tabWidth": 2,
+    "useTabs": false
+}

team-membership/README.md

@@ -0,0 +1,49 @@
+# TeamMembership action
+
+Check if the specified user belongs to a given team.
+
+It emits one outputs which are available via the `steps` [output context](https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#steps-context)
+
+## Documentation
+
+### Inputs
+| Input | Description | Required | Default |
+| ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- |
+| `organization` | Organization name | false | repo owner |
+| `username` | Username to check team membership | true |  |
+| `team` | Team to check the membership | true |  |
+
+### Outputs
+| Output | Description |
+| ------------------------------- | ----------------------------------------------------- |
+| `is_team_member` | A boolean indicating if a user belongs to a given team
+
+
+## Usage
+
+```yaml
+name: Action
+on: 
+  pull_request:
+  push:
+
+jobs:
+  check-right-permissions:
+    name: Check Right Permissions To Trigger This
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check Right Permissions To Trigger This
+        id: check_right_permissions
+        uses: stacks-network/actions/stacks-core/team-membership@main
+        with:
+          username: ${{ github.actor }}
+          team: 'Admin'
+
+      - name: Fail if the user does not have the right permissions
+        if: ${{ steps.check_right_permissions.outputs.is_team_member != 'true' }}
+        run: exit 1
+
+      - name: Proceed if the user has the right permissions
+        if: ${{ steps.check_right_permissions.outputs.is_team_member == 'true' }}
+        run: echo "User has the right permissions."
+```

team-membership/action.yml

@@ -0,0 +1,27 @@
+name: 'team-membership'
+description: 'Check if a user has membership in a specific team within an organization'
+branding:
+  icon: 'check'
+  color: 'gray-dark'
+inputs:
+  organization:
+    description: 'Organization name (default is repo owner)'
+    required: false
+    default: ${{ github.repository_owner }}
+  username:
+    description: 'Username to get teams or check team membership'
+    required: true
+  team:
+    description: 'Team to check the membership'
+    required: true
+  GITHUB_TOKEN:
+    description: 'GITHUB_TOKEN'
+    required: true
+
+outputs:
+  is_team_member:
+    description: 'Boolean indicating whether the user is a member of the specified team'
+
+runs:
+  using: 'node20'
+  main: 'dist/index.js'