Test Coverage Task

.github/workflows/intelligent-tests.yml

@@ -0,0 +1,31 @@
+name: test-core-ivy
+on:
+  push:
+  workflow_dispatch:
+permissions:
+  actions: read
+jobs:
+   run_tests:
+     runs-on: ubuntu-latest
+     steps:
+       - name: Checkout Ivy 🛎
+         uses: actions/checkout@v2
+         with:
+           path: ivy
+           persist-credentials: false
+           submodules: "recursive"
+
+       - name: Determine Tests
+         run: |
+           git clone https://github.com/unifyai/Mapping.git
+           cp Mapping/tests.pkl ivy/
+           cd ivy
+           python determine_tests.py
+
+#       - name: Run Functional-Core Tests
+#         id: tests
+#         if: steps.check_file_changed.outputs.changed == 'True' || steps.check_file_changed.conclusion == 'skipped'
+#         run: |
+#           cd ivy
+#           ./run_tests_CLI/test_ivy_core.sh ${{ matrix.backends }} test_${{ matrix.submodules }} ${{ secrets.REDIS_CONNECTION_URL }} ${{ secrets.REDIS_PASSWORD}}
+#         continue-on-error: true

determine_test_coverage.py

@@ -1,12 +1,23 @@
 import os
-from pprint import pprint
+from pydriller import Repository
 import pickle
+from tqdm import tqdm
 
 # Shared Map
 tests = {}
 
+# os.system("pytest --disable-pytest-warnings ivy_tests/test_ivy/ --my_test_dump true > test_names")
+test_names = []
+# with open("test_names") as f:
+#     i = 0
+#     for line in f:
+#         i += 1
+#         if i <= 5:
+#             continue
+#         test_names.append(line[:-1])
+#
+# test_names = test_names[:-3]
 
-# TODO: Add tests to this
 test_names = [
     "ivy_tests/test_ivy/test_functional/test_core/test_elementwise.py::test_abs",
 ]
@@ -54,9 +65,9 @@
 ]
 
 if __name__ == "__main__":
-    for test_name in test_names:
-        os.system(f"coverage run -m pytest {test_name}")
-        os.system("coverage annotate")
+    for test_name in tqdm(test_names):
+        os.system(f"coverage run -m pytest {test_name} --disable-warnings > coverage_output")
+        os.system("coverage annotate > coverage_output")
         for directory in directories:
             for file_name in os.listdir(directory):
                 if file_name.endswith("cover"):
@@ -75,7 +86,10 @@
         os.system("find . -name \\*cover -type f -delete")
 
 
-pprint(tests)
-
+commit_hash = ""
+for commit in Repository(".", order="reverse").traverse_commits():
+    commit_hash = commit.hash
+    break
+tests['commit'] = commit_hash
 with open("tests.pkl", "wb") as f:
     pickle.dump(tests, f)

determine_tests.py

@@ -6,10 +6,18 @@
 if __name__ == "__main__":
     with open("tests.pkl", "rb") as f:
         tests = pickle.load(f)
+    tests_to_run = set()
+    ref_commit_hash = tests['commit']
+    for commit in Repository('.', single=ref_commit_hash).traverse_commits():
+        ref_commit = commit._c_object
+        break
 
-    # pprint(tests)
     for commit in Repository(".", order="reverse").traverse_commits():
-        for file in commit.modified_files:
+        diff_index = ref_commit.diff(
+            commit._c_object, create_patch=True
+        )
+        modified_files = commit._parse_diff(diff_index)
+        for file in modified_files:
             file_name = file.new_path + ",cover"
             if file_name not in tests.keys():
                 continue
@@ -22,7 +30,6 @@
             added = added.difference(updated)
             deleted = deleted.difference(updated)
             # Now Update the Tests and compute the tests to run
-            tests_to_run = set()
             for line in deleted:
                 tests_to_run.update(tests_file[line])
             for line in deleted:
@@ -43,5 +50,7 @@
                 tests_to_run.update(tests_file[line])
             for line in added:
                 tests_to_run.update(tests_file[line])
-            print(tests_to_run)
         break
+
+    print(tests_to_run)
+    # Output Tests to a File

ivy_tests/test_ivy/conftest.py

@@ -128,7 +128,6 @@ def pytest_generate_tests(metafunc):
                     )
     metafunc.parametrize("device,f,compile_graph,implicit,fw", configs)
 
-
 @pytest.fixture(scope="session")
 def get_command_line_flags(request) -> Dict[str, bool]:
 
@@ -216,3 +215,12 @@ def pytest_addoption(parser):
         default=None,
         help="set deadline for testing one example",
     )
+    parser.addoption("--my_test_dump", action="store", default=None,
+        help="Print test items in my custom format")
+
+
+def pytest_collection_finish(session):
+    if session.config.option.my_test_dump is not None:
+        for item in session.items:
+            print('{}::{}'.format(item.fspath, item.name))
+        pytest.exit('Done!')