perf: added pinned python dependency count check

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-20.04]
-        python-version: ['3.8', '3.12']
+        python-version: ['3.12']
         toxenv: [python, quality]
 
     steps:

pylintrc

@@ -64,7 +64,7 @@
 # SERIOUSLY.
 #
 # ------------------------------
-# Generated by edx-lint version: 5.2.5
+# Generated by edx-lint version: 5.4.0
 # ------------------------------
 [MASTER]
 ignore = ,.git,.tox
@@ -259,6 +259,7 @@ enable =
 	useless-suppression,
 disable = 
 	bad-indentation,
+	broad-exception-raised,
 	consider-using-f-string,
 	duplicate-code,
 	file-ignored,
@@ -288,7 +289,9 @@ disable =
 	logging-fstring-interpolation,
 	django-not-available,
 	c-extension-no-member,
-	missing-timeout
+	missing-timeout,
+	possibly-used-before-assignment,
+	broad-exception-caught
 
 [REPORTS]
 output-format = text
@@ -383,6 +386,6 @@ ext-import-graph =
 int-import-graph = 
 
 [EXCEPTIONS]
-overgeneral-exceptions = Exception
+overgeneral-exceptions = builtins.Exception
 
-# 6a5f2b58c0741088bb34597f419f7419e326a841
+# 0a144c96a050dac3a5cbf676c19bf5e8fa63eac2

pylintrc_tweaks

@@ -3,8 +3,10 @@
 ignore+ = ,.git,.tox
 
 [MESSAGES CONTROL]
-disable+ = 
+disable+ =
 	django-not-available,
 	c-extension-no-member,
-	missing-timeout
+	missing-timeout,
+	possibly-used-before-assignment,
+	broad-exception-caught
 

repo_health/check_pinned_python_dependencies.py

@@ -0,0 +1,42 @@
+"""
+Counts the python dependencies which are pinned
+"""
+import os
+
+import pytest
+
+from repo_health import get_file_content
+
+module_dict_key = "pinned_python_dependencies"
+
+
+def get_dependencies_count(repo_path, file_name):
+    """
+   entry point to read requirements from constraints and common-constraints
+   @param repo_path:
+   @param file_name:
+   @return: number.
+   """
+    full_path = os.path.join(repo_path, "requirements/{0}".format(file_name))
+    content = get_file_content(full_path)
+    lines = content.split('\n')
+    dependency_count = 0
+    pinned_dependencies_marker = ['==', '>', '<']
+    for line in lines:
+        if line.startswith('#'):
+            continue
+        if any(marker in line for marker in pinned_dependencies_marker):
+            dependency_count += 1
+        else:
+            continue
+    return dependency_count
+
+
+@pytest.mark.edx_health
+def check_pinned_python_dependencies(repo_path, all_results):
+    """
+    We shall read constraints file
+    """
+    constraints_count = get_dependencies_count(repo_path, 'common_constraints.txt')
+    common_constraints_count = get_dependencies_count(repo_path, 'constraints.txt')
+    all_results[module_dict_key] = constraints_count + common_constraints_count

repo_health/check_ubuntufiles.py

@@ -232,7 +232,7 @@ def get_playbook_data(self, playbook_path):
             packages = self._prepare_data(list(packages))
             return packages
 
-        except Exception as exc:  # pylint: disable=useless-suppression
+        except Exception as exc:
             logger.exception("Following error occurred while parsing yml playbook (%s) in configuration repo: %s",
                              playbook_path, exc)
             return []

repo_health/utils.py

@@ -136,7 +136,7 @@ def get_release_tags(repo_dir):
             return all_tags_list[:all_tags_list.index(latest_tag) + 1]
         else:
             return None
-    except Exception as ex:
+    except Exception as ex:  # pylint: disable=broad-exception-caught
         print(str(ex))
         return None
 

requirements/base.txt

@@ -1,29 +1,27 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    make upgrade
 #
-aiohttp==3.9.3
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.10.10
     # via
     #   github-py
     #   pytest-aiohttp
 aiosignal==1.3.1
     # via aiohttp
-async-timeout==4.0.3
-    # via aiohttp
-attrs==23.2.0
+attrs==24.2.0
     # via aiohttp
-cachetools==5.3.3
+cachetools==5.5.0
     # via google-auth
-certifi==2024.2.2
+certifi==2024.8.30
     # via requests
-charset-normalizer==3.3.2
+charset-normalizer==3.4.0
     # via requests
 dockerfile==3.3.1
     # via -r requirements/base.in
-exceptiongroup==1.2.0
-    # via pytest
 frozenlist==1.4.1
     # via
     #   aiohttp
@@ -32,25 +30,25 @@ gitdb==4.0.11
     # via gitpython
 github-py @ git+https://github.com/ShineyDev/github.py.git@8e3514010eb35a91e012f2935f48138b3c35cef6
     # via -r requirements/github.in
-gitpython==3.1.42
+gitpython==3.1.43
     # via pytest-repo-health
-google-auth==2.28.1
+google-auth==2.35.0
     # via
     #   google-auth-oauthlib
     #   gspread
-google-auth-oauthlib==1.2.0
+google-auth-oauthlib==1.2.1
     # via gspread
 gspread==5.11.3
     # via
     #   -c requirements/constraints.txt
     #   -r requirements/base.in
-idna==3.6
+idna==3.10
     # via
     #   requests
     #   yarl
 iniconfig==2.0.0
     # via pytest
-multidict==6.0.5
+multidict==6.1.0
     # via
     #   aiohttp
     #   yarl
@@ -60,15 +58,17 @@ packaging==21.3
     # via
     #   -c requirements/constraints.txt
     #   pytest
-pluggy==1.4.0
+pluggy==1.5.0
     # via pytest
-pyasn1==0.5.1
+propcache==0.2.0
+    # via yarl
+pyasn1==0.6.1
     # via
     #   pyasn1-modules
     #   rsa
-pyasn1-modules==0.3.0
+pyasn1-modules==0.4.1
     # via google-auth
-pyparsing==3.1.2
+pyparsing==3.1.4
     # via packaging
 pytest==8.0.2
     # via
@@ -78,29 +78,27 @@ pytest==8.0.2
     #   pytest-repo-health
 pytest-aiohttp==1.0.5
     # via pytest-repo-health
-pytest-asyncio==0.23.5
+pytest-asyncio==0.23.8
     # via
     #   -r requirements/base.in
     #   pytest-aiohttp
 pytest-repo-health==3.0.2
     # via -r requirements/base.in
-pyyaml==6.0.1
+pyyaml==6.0.2
     # via
     #   -r requirements/base.in
     #   pytest-repo-health
-requests==2.31.0
+requests==2.32.3
     # via requests-oauthlib
-requests-oauthlib==1.3.1
+requests-oauthlib==2.0.0
     # via google-auth-oauthlib
 rsa==4.9
     # via google-auth
 smmap==5.0.1
     # via gitdb
 toml==0.10.2
     # via -r requirements/base.in
-tomli==2.0.1
-    # via pytest
-urllib3==2.2.1
+urllib3==2.2.3
     # via requests
-yarl==1.9.4
+yarl==1.14.0
     # via aiohttp

requirements/ci.txt

@@ -1,18 +1,18 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    make upgrade
 #
-cachetools==5.3.3
+cachetools==5.5.0
     # via tox
 chardet==5.2.0
     # via tox
 colorama==0.4.6
     # via tox
-distlib==0.3.8
+distlib==0.3.9
     # via virtualenv
-filelock==3.13.1
+filelock==3.16.1
     # via
     #   tox
     #   virtualenv
@@ -21,21 +21,17 @@ packaging==21.3
     #   -c requirements/constraints.txt
     #   pyproject-api
     #   tox
-platformdirs==4.2.0
+platformdirs==4.3.6
     # via
     #   tox
     #   virtualenv
-pluggy==1.4.0
+pluggy==1.5.0
     # via tox
-pyparsing==3.1.2
+pyparsing==3.1.4
     # via packaging
 pyproject-api==1.5.0
     # via tox
-tomli==2.0.1
-    # via
-    #   pyproject-api
-    #   tox
 tox==4.0.0
     # via -r requirements/ci.in
-virtualenv==20.25.1
+virtualenv==20.26.6
     # via tox

requirements/common_constraints.txt

@@ -18,7 +18,15 @@ Django<5.0
 
 # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
 # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
+# See https://github.com/openedx/edx-platform/issues/35126 for more info
 elasticsearch<7.14.0
 
 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
 django-simple-history==3.0.0
+
+# Cause: https://github.com/openedx/event-tracking/pull/290
+# event-tracking 2.4.1 upgrades to pymongo 4.4.0 which is not supported on edx-platform.
+# We will pin event-tracking to do not break existing installations
+# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586
+# has been resolved and edx-platform is running with pymongo>=4.4.0
+event-tracking<2.4.1

requirements/constraints.txt

@@ -15,13 +15,18 @@
 # greater version breaking test.
 packaging==21.3
 
-# gspread==5.12.0 contains breaking changes 
+# gspread==5.12.0 contains breaking changes
 # which break the scheduled repo health job
 gspread<5.12.0
 
 # pytest==8.1.0 breaks test plugin [needs to be investigated separately]
 pytest<8.1.0
 
-# Needed for Python 3.12 compatibility. 
+# Needed for Python 3.12 compatibility.
 # Can be removed once support for Python<3.12 is dropped.
 backports-zoneinfo==0.2.1; python_version < "3.9"
+
+# date added: 13-09-24
+# setuptools > 70.3.0 breaks the repo health workflow link: https://github.com/edx/repo-health-data/actions/runs/10846619044/job/30103106806
+# Issue to remove the pin https://github.com/openedx/edx-repo-health/issues/523
+setuptools==70.3.0