[ci][cov] Update coverage configs and streamline report generation

.github/workflows/commitflow-py3.yml

@@ -61,21 +61,19 @@ jobs:
 
     - name: run tests
       run: |
-        mkdir -p test-reports  # Ensure the directory exists
-        mkdir -p code-coverage-reports  # Ensure the directory exists
-        ./build/env/bin/pytest --cov=about --cov=beeswax --cov=filebrowser --cov=hbase --cov=help --cov=hive --cov=impala --cov=jobbrowser --cov=jobsub --cov=metastore --cov=oozie --cov=pig --cov=proxy --cov=rdbms --cov=search --cov=security --cov=spark --cov=sqoop --cov=useradmin --cov=zookeeper --cov=desktop --cov=aws --cov=azure --cov=dashboard --cov=hadoop --cov=indexer --cov=kafka --cov=libanalyze --cov=liboauth --cov=liboozie --cov=librdbms --cov=libsaml --cov=libsentry --cov=libsolr --cov=libzookeeper --cov=metadata --cov=notebook --cov-report=html:code-coverage-report --html=test-reports/report_${{ matrix.python-version }}.html --self-contained-html
+        ./build/env/bin/pytest
 
     - name: upload pytest report
       uses: actions/upload-artifact@v4
       with:
-        name: test-reports-${{ matrix.python-version }}
-        path: test-reports
+        name: hue-pytest-report-${{ matrix.python-version }}
+        path: reports/pytest
 
     - name: upload code coverage report
       uses: actions/upload-artifact@v4
       with:
-        name: code-coverage-report-${{ matrix.python-version }}
-        path: code-coverage-report
+        name: hue-code-cov-report-${{ matrix.python-version }}
+        path: reports/code-cov
 
     - name: Check and comment if no unit test files are modified
       if: matrix.python-version == '3.11'

.gitignore

@@ -61,10 +61,12 @@ desktop/core/ext-py/lxml/src/lxml/lxml-version.h
 # Nose --with-id's file
 /.noseids
 
-# Coverage product
-/.coverage
-/coverage.xml
-/coverage
+# Coverage reports
+.coverage*
+coverage.xml
+
+# Pytest reports
+report.html
 
 # Common utility alias
 /d
@@ -98,7 +100,6 @@ desktop/desktop-test.db
 ext/hadoop/
 ext/hive/
 ext/oozie/
-report.html
 oozie.sql
 tools/jenkins/ext/
 tools/jenkins/oozie.sql

pyproject.toml

@@ -1,31 +1,106 @@
 [tool.pytest.ini_options]
 DJANGO_SETTINGS_MODULE = "desktop.settings"
 console_output_style = "count"
-addopts = "-v -m 'not integration' --strict-markers --disable-warnings --html=report.html --self-contained-html"
+addopts = """
+-v \
+-m \
+'not integration' \
+--strict-markers \
+--disable-warnings \
+--html=reports/pytest/report.html \
+--self-contained-html \
+--cov=./ \
+--cov-report html:reports/code-cov/coverage-html \
+--cov-report xml:reports/code-cov/coverage.xml \
+"""
 norecursedirs = "desktop/core/ext-py3"
 python_files = "tests.py test_*.py *_tests.py tests_* *_test.py"
 markers = [
     "integration: live server based tests",
-    "requires_hadoop: live hadoop cluster based tests"
+    "requires_hadoop: live hadoop cluster based tests",
 ]
 
+[tool.coverage.run]
+branch = true
+omit = [
+    "*/ext-py3/*",
+    "desktop/core/src/desktop/lib/wsgiserver.py",
+    "*/migrations/*",
+    "*/org_migrations/*",
+    "*/old_migrations/*",
+    "*/gen-py/*",
+    "desktop/core/src/desktop/app_template/src/app_name/*",
+    "desktop/core/src/desktop/app_template_proxy/src/app_name/*",
+    "*conf.py",
+    "*settings.py",
+    "*urls.py",
+    "*metrics.py",
+    "*tests.py*",
+    "test_*.py",
+    "*_tests.py",
+    "tests_*.py",
+    "*_test.py",
+]
+source = [
+    "about",
+    "beeswax",
+    "filebrowser",
+    "hbase",
+    "help",
+    "hive",
+    "impala",
+    "jobbrowser",
+    "jobsub",
+    "metastore",
+    "oozie",
+    "pig",
+    "proxy",
+    "rdbms",
+    "search",
+    "security",
+    "spark",
+    "sqoop",
+    "useradmin",
+    "zookeeper",
+    "desktop",
+    "aws",
+    "azure",
+    "dashboard",
+    "hadoop",
+    "indexer",
+    "kafka",
+    "libanalyze",
+    "liboauth",
+    "liboozie",
+    "librdbms",
+    "libsaml",
+    "libsentry",
+    "libsolr",
+    "libzookeeper",
+    "metadata",
+    "notebook",
+]
+
+[tool.coverage.report]
+precision = 2
+
 [tool.ruff]
 target-version = "py38"
 line-length = 140
 indent-width = 2
 force-exclude = true
 extend-exclude = [
-  "*/ext-py3/*",
-  "desktop/core/src/desktop/lib/wsgiserver.py",
-  "*/migrations/*",
-  "apps/oozie/src/oozie/tests.py",
-  "tools/ops/",
-  "tools/ace-editor/",
-  "*/gen-py/*",
-  "*/org_migrations/*",
-  "*/old_migrations/*",
-  "desktop/core/src/desktop/app_template/src/app_name/*",
-  "desktop/core/src/desktop/app_template_proxy/src/app_name/*"
+    "*/ext-py3/*",
+    "desktop/core/src/desktop/lib/wsgiserver.py",
+    "*/migrations/*",
+    "apps/oozie/src/oozie/tests.py",
+    "tools/ops/",
+    "tools/ace-editor/",
+    "*/gen-py/*",
+    "*/org_migrations/*",
+    "*/old_migrations/*",
+    "desktop/core/src/desktop/app_template/src/app_name/*",
+    "desktop/core/src/desktop/app_template_proxy/src/app_name/*",
 ]
 
 [tool.ruff.lint]
@@ -35,13 +110,7 @@ select = [
     "W", # pycodestyle warnings
     "I", # isort
 ]
-ignore = [
-    "E111",
-    "E114",
-    "E117",
-    "W191",
-    "E731"
-]
+ignore = ["E111", "E114", "E117", "W191", "E731"]
 
 [tool.ruff.lint.per-file-ignores]
 "desktop/libs/notebook/src/notebook/connectors/altus.py" = ["E501"]
@@ -101,5 +170,5 @@ known-first-party = [
     "libsolr",
     "libzookeeper",
     "metadata",
-    "notebook"
-]
+    "notebook",
+]