coverage via coveralls in CircleCI (#515)

Summary:
## Types of changes

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
- [ ] Docs change / refactoring / dependency upgrade

(None of the above - changes are made to CI pipeline)

## Motivation and Context / Related issue

The objective was to have code coverage made [publicly accessible](#424) via https://coveralls.io
To achieve this, the following changes were made:
1. move existing tests running via `pytest` & `unittest` to run via [`coverage`](https://coverage.readthedocs.io)
2. add dependency of `coverage` in `dev_requirements.txt`, followed by `coveralls` in CircleCI config
3. minor refactoring of tasks to prepare for upload to `coveralls`
4. upload coverage reports generated in `parallel` to `coveralls` as CircleCI tasks
5. at the end of workflow, signal `coveralls` to combine all `parallel` uploads under single run

## How Has This Been Tested (if it applies)

Testing changes made to CircleCI config is tricky. Most of the code was tested via opening this PR and letting CircleCI run on this new config. Some obscure parts were tested in a personal CircleCI environment.

## Checklist

- [ ] The documentation is up-to-date with the changes I made.
- [x] I have read the **CONTRIBUTING** document and completed the CLA (see **CONTRIBUTING**).
- [x] All tests passed, and additional code has been covered with new tests.

Pull Request resolved: #515

Reviewed By: ashkan-software

Differential Revision: D39795415

Pulled By: ashkan-software

fbshipit-source-id: 82830bfea917bed7efcb15d854f7e1af34802dbb

.circleci/config.yml

@@ -20,6 +20,8 @@ commands:
             echo "In venv: $(pyenv local) - $(python -V), $(pip -V)"
             sudo "$(which python)" -m pip install --upgrade pip
             sudo "$(which python)" -m pip install pytest
+            sudo "$(which python)" -m pip install coverage
+            sudo "$(which python)" -m pip install coveralls
 
   run_nvidia_smi:
     description: "Prints GPU capabilities from nvidia-smi"
@@ -92,13 +94,34 @@ commands:
           no_output_timeout: 1h
           command: |
             mkdir unittest-reports
-            python -m pytest --doctest-modules -p conftest --junitxml=unittest-reports/junit.xml opacus
+            coverage run -m pytest --doctest-modules -p conftest --junitxml=unittest-reports/junit.xml opacus
+            coverage report -i -m
 
       - store_test_results:
           path: unittest-reports
       - store_artifacts:
           path: unittest-reports
 
+  command_unit_tests_multi_gpu:
+    description: "Run multi gpu unit tests"
+    steps:
+      - run:
+          name: "Unit test multi_gpu"
+          no_output_timeout: 1h
+          command: |
+            mkdir unittest-multigpu-reports
+            coverage run -m unittest opacus.tests.multigpu_gradcheck.GradientComputationTest.test_gradient_correct
+            coverage report -i -m
+
+  coveralls_upload_parallel:
+    description: "upload coverage to coveralls"
+    steps:
+      - run:
+          name: "coveralls upload"
+          no_output_timeout: 5m
+          command: |
+            pip install coveralls --user
+            COVERALLS_PARALLEL=true COVERALLS_FLAG_NAME="${CIRCLE_JOB}" coveralls
 
   mnist_integration_test:
     description: "Runs MNIST example end to end"
@@ -300,6 +323,7 @@ jobs:
       - checkout
       - pip_dev_install
       - unit_tests
+      - coveralls_upload_parallel
 
   unittest_py38_torch_release:
     docker:
@@ -308,6 +332,7 @@ jobs:
       - checkout
       - pip_dev_install
       - unit_tests
+      - coveralls_upload_parallel
 
   unittest_py39_torch_release:
     docker:
@@ -316,6 +341,7 @@ jobs:
       - checkout
       - pip_dev_install
       - unit_tests
+      - coveralls_upload_parallel
 
   unittest_py39_torch_nightly:
     docker:
@@ -325,6 +351,7 @@ jobs:
       - pip_dev_install:
           args: "-n"
       - unit_tests
+      - coveralls_upload_parallel
 
   integrationtest_py37_torch_release_cpu:
     docker:
@@ -449,12 +476,8 @@ jobs:
       - py_3_7_setup
       - pip_dev_install
       - run_nvidia_smi
-      - run:
-          name: "Unit test multi_gpu"
-          no_output_timeout: 1h
-          command: |
-            mkdir unittest-multigpu-reports
-            python -m unittest opacus.tests.multigpu_gradcheck.GradientComputationTest.test_gradient_correct
+      - command_unit_tests_multi_gpu
+      - coveralls_upload_parallel
 
 
   auto_deploy_site:
@@ -469,6 +492,17 @@ jobs:
       - configure_docusaurus_bot
       - deploy_site
 
+  finish_coveralls_parallel:
+    docker:
+      - image: cimg/python:3.9
+    steps:
+      - run:
+          name: "finish coveralls parallel"
+          no_output_timeout: 5m
+          command: |
+            pip install coveralls --user
+            coveralls --finish
+
 
 aliases:
 
@@ -503,6 +537,14 @@ workflows:
           filters: *exclude_ghpages
       - integrationtest_py37_torch_release_cuda:
           filters: *exclude_ghpages
+      - finish_coveralls_parallel:
+          filters: *exclude_ghpages
+          requires:
+            - unittest_py37_torch_release
+            - unittest_py38_torch_release
+            - unittest_py39_torch_release
+            - unittest_py39_torch_nightly
+            - unittest_multi_gpu
 
   nightly:
     when:
@@ -518,6 +560,10 @@ workflows:
           filters: *exclude_ghpages
       - micro_benchmarks_py37_torch_release_cuda:
           filters: *exclude_ghpages
+      - finish_coveralls_parallel:
+          filters: *exclude_ghpages
+          requires:
+            - unittest_py39_torch_nightly
 
   website_deployment:
     when:

README.md

@@ -3,6 +3,7 @@
 <hr/>
 
 [![CircleCI](https://circleci.com/gh/pytorch/opacus.svg?style=svg)](https://circleci.com/gh/pytorch/opacus)
+[![Coverage Status](https://coveralls.io/repos/github/pytorch/opacus/badge.svg?branch=main)](https://coveralls.io/github/pytorch/opacus?branch=main)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](CONTRIBUTING.md)
 [![License](https://img.shields.io/badge/license-apache2-green.svg)](LICENSE)
 

dev_requirements.txt

@@ -17,3 +17,4 @@ scikit-learn
 pytorch-lightning
 lightning-bolts
 jsonargparse[signatures]>=3.19.3 # required for Lightning CLI
+coverage