From 3bebe5641c48f6211adfb2f0ef297dde4beca0f5 Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Thu, 11 Jul 2024 13:44:20 +0500 Subject: [PATCH] perf: added pinned python dependency count check --- .github/workflows/ci.yml | 2 +- pylintrc | 11 +- pylintrc_tweaks | 6 +- .../check_pinned_python_dependencies.py | 42 +++++ repo_health/check_ubuntufiles.py | 2 +- repo_health/utils.py | 2 +- requirements/base.txt | 52 +++---- requirements/ci.txt | 20 +-- requirements/common_constraints.txt | 8 + requirements/constraints.txt | 9 +- requirements/dev.txt | 147 +++++++----------- requirements/doc.txt | 120 +++++++------- requirements/pip-tools.txt | 19 +-- requirements/pip.txt | 12 +- requirements/quality.txt | 111 ++++++------- requirements/test.txt | 69 ++++---- scripts/repo-health-script.sh | 4 +- .../requirements/common_constraints.txt | 24 +++ .../requirements/constraints.txt | 25 +++ 19 files changed, 358 insertions(+), 327 deletions(-) create mode 100644 repo_health/check_pinned_python_dependencies.py create mode 100644 tests/fake_repos/python_pinned_dependencies/requirements/common_constraints.txt create mode 100644 tests/fake_repos/python_pinned_dependencies/requirements/constraints.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7be86ba1..8a73a006 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04] - python-version: ['3.8', '3.12'] + python-version: ['3.12'] toxenv: [python, quality] steps: diff --git a/pylintrc b/pylintrc index 22cdbf7e..c2e1431b 100644 --- a/pylintrc +++ b/pylintrc @@ -64,7 +64,7 @@ # SERIOUSLY. # # ------------------------------ -# Generated by edx-lint version: 5.2.5 +# Generated by edx-lint version: 5.4.0 # ------------------------------ [MASTER] ignore = ,.git,.tox @@ -259,6 +259,7 @@ enable = useless-suppression, disable = bad-indentation, + broad-exception-raised, consider-using-f-string, duplicate-code, file-ignored, @@ -288,7 +289,9 @@ disable = logging-fstring-interpolation, django-not-available, c-extension-no-member, - missing-timeout + missing-timeout, + possibly-used-before-assignment, + broad-exception-caught [REPORTS] output-format = text @@ -383,6 +386,6 @@ ext-import-graph = int-import-graph = [EXCEPTIONS] -overgeneral-exceptions = Exception +overgeneral-exceptions = builtins.Exception -# 6a5f2b58c0741088bb34597f419f7419e326a841 +# 0a144c96a050dac3a5cbf676c19bf5e8fa63eac2 diff --git a/pylintrc_tweaks b/pylintrc_tweaks index 8e55ab68..b42cc34c 100644 --- a/pylintrc_tweaks +++ b/pylintrc_tweaks @@ -3,8 +3,10 @@ ignore+ = ,.git,.tox [MESSAGES CONTROL] -disable+ = +disable+ = django-not-available, c-extension-no-member, - missing-timeout + missing-timeout, + possibly-used-before-assignment, + broad-exception-caught diff --git a/repo_health/check_pinned_python_dependencies.py b/repo_health/check_pinned_python_dependencies.py new file mode 100644 index 00000000..ed9fdb55 --- /dev/null +++ b/repo_health/check_pinned_python_dependencies.py @@ -0,0 +1,42 @@ +""" +Counts the python dependencies which are pinned +""" +import os + +import pytest + +from repo_health import get_file_content + +module_dict_key = "pinned_python_dependencies" + + +def get_dependencies_count(repo_path, file_name): + """ + entry point to read requirements from constraints and common-constraints + @param repo_path: + @param file_name: + @return: number. + """ + full_path = os.path.join(repo_path, "requirements/{0}".format(file_name)) + content = get_file_content(full_path) + lines = content.split('\n') + dependency_count = 0 + pinned_dependencies_marker = ['==', '>', '<'] + for line in lines: + if line.startswith('#'): + continue + if any(marker in line for marker in pinned_dependencies_marker): + dependency_count += 1 + else: + continue + return dependency_count + + +@pytest.mark.edx_health +def check_pinned_python_dependencies(repo_path, all_results): + """ + We shall read constraints file + """ + constraints_count = get_dependencies_count(repo_path, 'common_constraints.txt') + common_constraints_count = get_dependencies_count(repo_path, 'constraints.txt') + all_results[module_dict_key] = constraints_count + common_constraints_count diff --git a/repo_health/check_ubuntufiles.py b/repo_health/check_ubuntufiles.py index c9741be2..243f4523 100644 --- a/repo_health/check_ubuntufiles.py +++ b/repo_health/check_ubuntufiles.py @@ -232,7 +232,7 @@ def get_playbook_data(self, playbook_path): packages = self._prepare_data(list(packages)) return packages - except Exception as exc: # pylint: disable=useless-suppression + except Exception as exc: logger.exception("Following error occurred while parsing yml playbook (%s) in configuration repo: %s", playbook_path, exc) return [] diff --git a/repo_health/utils.py b/repo_health/utils.py index 8b5f7e34..71b82dde 100644 --- a/repo_health/utils.py +++ b/repo_health/utils.py @@ -136,7 +136,7 @@ def get_release_tags(repo_dir): return all_tags_list[:all_tags_list.index(latest_tag) + 1] else: return None - except Exception as ex: + except Exception as ex: # pylint: disable=broad-exception-caught print(str(ex)) return None diff --git a/requirements/base.txt b/requirements/base.txt index bbd10238..4a2939a0 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,29 +1,27 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -aiohttp==3.9.3 +aiohappyeyeballs==2.4.3 + # via aiohttp +aiohttp==3.10.10 # via # github-py # pytest-aiohttp aiosignal==1.3.1 # via aiohttp -async-timeout==4.0.3 - # via aiohttp -attrs==23.2.0 +attrs==24.2.0 # via aiohttp -cachetools==5.3.3 +cachetools==5.5.0 # via google-auth -certifi==2024.2.2 +certifi==2024.8.30 # via requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via requests dockerfile==3.3.1 # via -r requirements/base.in -exceptiongroup==1.2.0 - # via pytest frozenlist==1.4.1 # via # aiohttp @@ -32,25 +30,25 @@ gitdb==4.0.11 # via gitpython github-py @ git+https://github.com/ShineyDev/github.py.git@8e3514010eb35a91e012f2935f48138b3c35cef6 # via -r requirements/github.in -gitpython==3.1.42 +gitpython==3.1.43 # via pytest-repo-health -google-auth==2.28.1 +google-auth==2.35.0 # via # google-auth-oauthlib # gspread -google-auth-oauthlib==1.2.0 +google-auth-oauthlib==1.2.1 # via gspread gspread==5.11.3 # via # -c requirements/constraints.txt # -r requirements/base.in -idna==3.6 +idna==3.10 # via # requests # yarl iniconfig==2.0.0 # via pytest -multidict==6.0.5 +multidict==6.1.0 # via # aiohttp # yarl @@ -60,15 +58,17 @@ packaging==21.3 # via # -c requirements/constraints.txt # pytest -pluggy==1.4.0 +pluggy==1.5.0 # via pytest -pyasn1==0.5.1 +propcache==0.2.0 + # via yarl +pyasn1==0.6.1 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.1 # via google-auth -pyparsing==3.1.2 +pyparsing==3.1.4 # via packaging pytest==8.0.2 # via @@ -78,19 +78,19 @@ pytest==8.0.2 # pytest-repo-health pytest-aiohttp==1.0.5 # via pytest-repo-health -pytest-asyncio==0.23.5 +pytest-asyncio==0.23.8 # via # -r requirements/base.in # pytest-aiohttp pytest-repo-health==3.0.2 # via -r requirements/base.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/base.in # pytest-repo-health -requests==2.31.0 +requests==2.32.3 # via requests-oauthlib -requests-oauthlib==1.3.1 +requests-oauthlib==2.0.0 # via google-auth-oauthlib rsa==4.9 # via google-auth @@ -98,9 +98,7 @@ smmap==5.0.1 # via gitdb toml==0.10.2 # via -r requirements/base.in -tomli==2.0.1 - # via pytest -urllib3==2.2.1 +urllib3==2.2.3 # via requests -yarl==1.9.4 +yarl==1.14.0 # via aiohttp diff --git a/requirements/ci.txt b/requirements/ci.txt index 1535f01c..76a9e111 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,18 +1,18 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -cachetools==5.3.3 +cachetools==5.5.0 # via tox chardet==5.2.0 # via tox colorama==0.4.6 # via tox -distlib==0.3.8 +distlib==0.3.9 # via virtualenv -filelock==3.13.1 +filelock==3.16.1 # via # tox # virtualenv @@ -21,21 +21,17 @@ packaging==21.3 # -c requirements/constraints.txt # pyproject-api # tox -platformdirs==4.2.0 +platformdirs==4.3.6 # via # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via tox -pyparsing==3.1.2 +pyparsing==3.1.4 # via packaging pyproject-api==1.5.0 # via tox -tomli==2.0.1 - # via - # pyproject-api - # tox tox==4.0.0 # via -r requirements/ci.in -virtualenv==20.25.1 +virtualenv==20.26.6 # via tox diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 0ab0ecd2..9a458f05 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -18,7 +18,15 @@ Django<5.0 # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html +# See https://github.com/openedx/edx-platform/issues/35126 for more info elasticsearch<7.14.0 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected django-simple-history==3.0.0 + +# Cause: https://github.com/openedx/event-tracking/pull/290 +# event-tracking 2.4.1 upgrades to pymongo 4.4.0 which is not supported on edx-platform. +# We will pin event-tracking to do not break existing installations +# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586 +# has been resolved and edx-platform is running with pymongo>=4.4.0 +event-tracking<2.4.1 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 9c752b38..e9d947da 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -15,13 +15,18 @@ # greater version breaking test. packaging==21.3 -# gspread==5.12.0 contains breaking changes +# gspread==5.12.0 contains breaking changes # which break the scheduled repo health job gspread<5.12.0 # pytest==8.1.0 breaks test plugin [needs to be investigated separately] pytest<8.1.0 -# Needed for Python 3.12 compatibility. +# Needed for Python 3.12 compatibility. # Can be removed once support for Python<3.12 is dropped. backports-zoneinfo==0.2.1; python_version < "3.9" + +# date added: 13-09-24 +# setuptools > 70.3.0 breaks the repo health workflow link: https://github.com/edx/repo-health-data/actions/runs/10846619044/job/30103106806 +# Issue to remove the pin https://github.com/openedx/edx-repo-health/issues/523 +setuptools==70.3.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index 3353eee3..d08e1687 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,10 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -aiohttp==3.9.3 +aiohappyeyeballs==2.4.3 + # via + # -r requirements/quality.txt + # aiohttp +aiohttp==3.10.10 # via # -r requirements/quality.txt # github-py @@ -13,39 +17,30 @@ aiosignal==1.3.1 # via # -r requirements/quality.txt # aiohttp -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/quality.txt # django -astroid==3.1.0 +astroid==3.3.5 # via # -r requirements/quality.txt # pylint # pylint-celery -async-timeout==4.0.3 +attrs==24.2.0 # via # -r requirements/quality.txt # aiohttp -attrs==23.2.0 - # via - # -r requirements/quality.txt - # aiohttp -backports-zoneinfo==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # -r requirements/quality.txt - # django -build==1.1.1 +build==1.2.2.post1 # via # -r requirements/pip-tools.txt # pip-tools -cachetools==5.3.3 +cachetools==5.5.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # google-auth # tox -certifi==2024.2.2 +certifi==2024.8.30 # via # -r requirements/quality.txt # requests @@ -54,7 +49,7 @@ chardet==5.2.0 # -r requirements/ci.txt # diff-cover # tox -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/quality.txt # requests @@ -70,7 +65,7 @@ click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint -code-annotations==1.6.0 +code-annotations==1.8.0 # via # -r requirements/quality.txt # edx-lint @@ -78,33 +73,29 @@ colorama==0.4.6 # via # -r requirements/ci.txt # tox -coverage[toml]==7.4.3 +coverage[toml]==7.6.2 # via # -r requirements/quality.txt # pytest-cov -diff-cover==8.0.3 +diff-cover==9.2.0 # via -r requirements/dev.in -dill==0.3.8 +dill==0.3.9 # via # -r requirements/quality.txt # pylint -distlib==0.3.8 +distlib==0.3.9 # via # -r requirements/ci.txt # virtualenv -django==4.2.11 +django==4.2.16 # via # -c requirements/common_constraints.txt # -r requirements/quality.txt dockerfile==3.3.1 # via -r requirements/quality.txt -edx-lint==5.3.6 +edx-lint==5.4.0 # via -r requirements/quality.txt -exceptiongroup==1.2.0 - # via - # -r requirements/quality.txt - # pytest -filelock==3.13.1 +filelock==3.16.1 # via # -r requirements/ci.txt # tox @@ -120,16 +111,16 @@ gitdb==4.0.11 # gitpython github-py @ git+https://github.com/ShineyDev/github.py.git@8e3514010eb35a91e012f2935f48138b3c35cef6 # via -r requirements/quality.txt -gitpython==3.1.42 +gitpython==3.1.43 # via # -r requirements/quality.txt # pytest-repo-health -google-auth==2.28.1 +google-auth==2.35.0 # via # -r requirements/quality.txt # google-auth-oauthlib # gspread -google-auth-oauthlib==1.2.0 +google-auth-oauthlib==1.2.1 # via # -r requirements/quality.txt # gspread @@ -137,15 +128,11 @@ gspread==5.11.3 # via # -c requirements/constraints.txt # -r requirements/quality.txt -idna==3.6 +idna==3.10 # via # -r requirements/quality.txt # requests # yarl -importlib-metadata==7.0.2 - # via - # -r requirements/pip-tools.txt - # build iniconfig==2.0.0 # via # -r requirements/quality.txt @@ -154,14 +141,14 @@ isort==5.13.2 # via # -r requirements/quality.txt # pylint -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/quality.txt # code-annotations # diff-cover markdown-it-py==3.0.0 # via rich -markupsafe==2.1.5 +markupsafe==3.0.1 # via # -r requirements/quality.txt # jinja2 @@ -171,7 +158,7 @@ mccabe==0.7.0 # pylint mdurl==0.1.2 # via markdown-it-py -multidict==6.0.5 +multidict==6.1.0 # via # -r requirements/quality.txt # aiohttp @@ -190,44 +177,48 @@ packaging==21.3 # pyproject-api # pytest # tox -pbr==6.0.0 +pbr==6.1.0 # via # -r requirements/quality.txt # stevedore pip-tools==7.4.1 # via -r requirements/pip-tools.txt -platformdirs==4.2.0 +platformdirs==4.3.6 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # diff-cover # pytest # tox -pyasn1==0.5.1 +propcache==0.2.0 + # via + # -r requirements/quality.txt + # yarl +pyasn1==0.6.1 # via # -r requirements/quality.txt # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.1 # via # -r requirements/quality.txt # google-auth -pycodestyle==2.11.1 +pycodestyle==2.12.1 # via -r requirements/quality.txt pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.17.2 +pygments==2.18.0 # via # diff-cover # rich -pylint==3.1.0 +pylint==3.3.1 # via # -r requirements/quality.txt # edx-lint @@ -238,7 +229,7 @@ pylint-celery==0.3 # via # -r requirements/quality.txt # edx-lint -pylint-django==2.5.5 +pylint-django==2.6.1 # via # -r requirements/quality.txt # edx-lint @@ -247,7 +238,7 @@ pylint-plugin-utils==0.8.2 # -r requirements/quality.txt # pylint-celery # pylint-django -pyparsing==3.1.2 +pyparsing==3.1.4 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt @@ -257,7 +248,7 @@ pyproject-api==1.5.0 # via # -r requirements/ci.txt # tox -pyproject-hooks==1.0.0 +pyproject-hooks==1.2.0 # via # -r requirements/pip-tools.txt # build @@ -274,11 +265,11 @@ pytest-aiohttp==1.0.5 # via # -r requirements/quality.txt # pytest-repo-health -pytest-asyncio==0.23.5 +pytest-asyncio==0.23.8 # via # -r requirements/quality.txt # pytest-aiohttp -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/quality.txt pytest-repo-health==3.0.2 # via -r requirements/quality.txt @@ -286,24 +277,24 @@ python-slugify==8.0.4 # via # -r requirements/quality.txt # code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/quality.txt # code-annotations # pytest-repo-health # responses -requests==2.31.0 +requests==2.32.3 # via # -r requirements/quality.txt # requests-oauthlib # responses -requests-oauthlib==1.3.1 +requests-oauthlib==2.0.0 # via # -r requirements/quality.txt # google-auth-oauthlib -responses==0.25.0 +responses==0.25.3 # via -r requirements/quality.txt -rich==13.7.1 +rich==13.9.2 # via -r requirements/dev.in rsa==4.9 # via @@ -321,11 +312,11 @@ snowballstemmer==2.2.0 # via # -r requirements/quality.txt # pydocstyle -sqlparse==0.4.4 +sqlparse==0.5.1 # via # -r requirements/quality.txt # django -stevedore==5.2.0 +stevedore==5.3.0 # via # -r requirements/quality.txt # code-annotations @@ -335,53 +326,29 @@ text-unidecode==1.3 # python-slugify toml==0.10.2 # via -r requirements/quality.txt -tomli==2.0.1 - # via - # -r requirements/ci.txt - # -r requirements/pip-tools.txt - # -r requirements/quality.txt - # build - # coverage - # pip-tools - # pylint - # pyproject-api - # pyproject-hooks - # pytest - # tox -tomlkit==0.12.4 +tomlkit==0.13.2 # via # -r requirements/quality.txt # pylint tox==4.0.0 # via -r requirements/ci.txt -typing-extensions==4.10.0 - # via - # -r requirements/quality.txt - # asgiref - # astroid - # pylint - # rich -urllib3==2.2.1 +urllib3==2.2.3 # via # -r requirements/quality.txt # requests # responses -virtualenv==20.25.1 +virtualenv==20.26.6 # via # -r requirements/ci.txt # tox -wheel==0.42.0 +wheel==0.44.0 # via # -r requirements/pip-tools.txt # pip-tools -yarl==1.9.4 +yarl==1.14.0 # via # -r requirements/quality.txt # aiohttp -zipp==3.17.0 - # via - # -r requirements/pip-tools.txt - # importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/doc.txt b/requirements/doc.txt index 8d0fac13..909a1b49 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,12 +1,16 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -accessible-pygments==0.0.4 +accessible-pygments==0.0.5 # via pydata-sphinx-theme -aiohttp==3.9.3 +aiohappyeyeballs==2.4.3 + # via + # -r requirements/test.txt + # aiohttp +aiohttp==3.10.10 # via # -r requirements/test.txt # github-py @@ -15,53 +19,45 @@ aiosignal==1.3.1 # via # -r requirements/test.txt # aiohttp -alabaster==0.7.13 +alabaster==0.7.16 # via sphinx -async-timeout==4.0.3 +attrs==24.2.0 # via # -r requirements/test.txt # aiohttp -attrs==23.2.0 - # via - # -r requirements/test.txt - # aiohttp -babel==2.14.0 +babel==2.16.0 # via # pydata-sphinx-theme # sphinx beautifulsoup4==4.12.3 # via pydata-sphinx-theme -cachetools==5.3.3 +cachetools==5.5.0 # via # -r requirements/test.txt # google-auth -certifi==2024.2.2 +certifi==2024.8.30 # via # -r requirements/test.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/test.txt # requests -coverage[toml]==7.4.3 +coverage[toml]==7.6.2 # via # -r requirements/test.txt # pytest-cov -doc8==1.1.1 +doc8==1.1.2 # via -r requirements/doc.in dockerfile==3.3.1 # via -r requirements/test.txt -docutils==0.19 +docutils==0.21.2 # via # doc8 # pydata-sphinx-theme # readme-renderer # restructuredtext-lint # sphinx -exceptiongroup==1.2.0 - # via - # -r requirements/test.txt - # pytest frozenlist==1.4.1 # via # -r requirements/test.txt @@ -73,16 +69,16 @@ gitdb==4.0.11 # gitpython github-py @ git+https://github.com/ShineyDev/github.py.git@8e3514010eb35a91e012f2935f48138b3c35cef6 # via -r requirements/test.txt -gitpython==3.1.42 +gitpython==3.1.43 # via # -r requirements/test.txt # pytest-repo-health -google-auth==2.28.1 +google-auth==2.35.0 # via # -r requirements/test.txt # google-auth-oauthlib # gspread -google-auth-oauthlib==1.2.0 +google-auth-oauthlib==1.2.1 # via # -r requirements/test.txt # gspread @@ -90,29 +86,27 @@ gspread==5.11.3 # via # -c requirements/constraints.txt # -r requirements/test.txt -idna==3.6 +idna==3.10 # via # -r requirements/test.txt # requests # yarl imagesize==1.4.1 # via sphinx -importlib-metadata==7.0.2 - # via sphinx iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -jinja2==3.1.3 +jinja2==3.1.4 # via sphinx -markupsafe==2.1.5 +markupsafe==3.0.1 # via jinja2 -multidict==6.0.5 +multidict==6.1.0 # via # -r requirements/test.txt # aiohttp # yarl -nh3==0.2.15 +nh3==0.2.18 # via readme-renderer oauthlib==3.2.2 # via @@ -125,31 +119,35 @@ packaging==21.3 # pydata-sphinx-theme # pytest # sphinx -pbr==6.0.0 +pbr==6.1.0 # via stevedore -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/test.txt # pytest -pyasn1==0.5.1 +propcache==0.2.0 + # via + # -r requirements/test.txt + # yarl +pyasn1==0.6.1 # via # -r requirements/test.txt # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.1 # via # -r requirements/test.txt # google-auth -pydata-sphinx-theme==0.14.4 +pydata-sphinx-theme==0.15.4 # via sphinx-book-theme -pygments==2.17.2 +pygments==2.18.0 # via # accessible-pygments # doc8 # pydata-sphinx-theme # readme-renderer # sphinx -pyparsing==3.1.2 +pyparsing==3.1.4 # via # -r requirements/test.txt # packaging @@ -165,34 +163,32 @@ pytest-aiohttp==1.0.5 # via # -r requirements/test.txt # pytest-repo-health -pytest-asyncio==0.23.5 +pytest-asyncio==0.23.8 # via # -r requirements/test.txt # pytest-aiohttp -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.txt pytest-repo-health==3.0.2 # via -r requirements/test.txt -pytz==2024.1 - # via babel -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/test.txt # pytest-repo-health # responses -readme-renderer==43.0 +readme-renderer==44.0 # via -r requirements/doc.in -requests==2.31.0 +requests==2.32.3 # via # -r requirements/test.txt # requests-oauthlib # responses # sphinx -requests-oauthlib==1.3.1 +requests-oauthlib==2.0.0 # via # -r requirements/test.txt # google-auth-oauthlib -responses==0.25.0 +responses==0.25.3 # via -r requirements/test.txt restructuredtext-lint==1.4.0 # via doc8 @@ -206,47 +202,39 @@ smmap==5.0.1 # gitdb snowballstemmer==2.2.0 # via sphinx -soupsieve==2.5 +soupsieve==2.6 # via beautifulsoup4 -sphinx==6.2.1 +sphinx==7.3.7 # via # -r requirements/doc.in # pydata-sphinx-theme # sphinx-book-theme -sphinx-book-theme==1.0.1 +sphinx-book-theme==1.1.3 # via -r requirements/doc.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx -stevedore==5.2.0 +stevedore==5.3.0 # via doc8 toml==0.10.2 # via -r requirements/test.txt -tomli==2.0.1 - # via - # -r requirements/test.txt - # coverage - # doc8 - # pytest -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via pydata-sphinx-theme -urllib3==2.2.1 +urllib3==2.2.3 # via # -r requirements/test.txt # requests # responses -yarl==1.9.4 +yarl==1.14.0 # via # -r requirements/test.txt # aiohttp -zipp==3.17.0 - # via importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 1ae42163..60ad3072 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,36 +1,27 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -build==1.1.1 +build==1.2.2.post1 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==7.0.2 - # via build packaging==21.3 # via # -c requirements/constraints.txt # build pip-tools==7.4.1 # via -r requirements/pip-tools.in -pyparsing==3.1.2 +pyparsing==3.1.4 # via packaging -pyproject-hooks==1.0.0 +pyproject-hooks==1.2.0 # via # build # pip-tools -tomli==2.0.1 - # via - # build - # pip-tools - # pyproject-hooks -wheel==0.42.0 +wheel==0.44.0 # via pip-tools -zipp==3.17.0 - # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/pip.txt b/requirements/pip.txt index 66656035..38da142a 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,14 +1,16 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -wheel==0.42.0 +wheel==0.44.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: -pip==24.0 - # via -r requirements/pip.in -setuptools==69.1.1 +pip==24.2 # via -r requirements/pip.in +setuptools==70.3.0 + # via + # -c requirements/constraints.txt + # -r requirements/pip.in diff --git a/requirements/quality.txt b/requirements/quality.txt index f7ce7800..36289532 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,10 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -aiohttp==3.9.3 +aiohappyeyeballs==2.4.3 + # via + # -r requirements/test.txt + # aiohttp +aiohttp==3.10.10 # via # -r requirements/test.txt # github-py @@ -13,33 +17,25 @@ aiosignal==1.3.1 # via # -r requirements/test.txt # aiohttp -asgiref==3.7.2 +asgiref==3.8.1 # via django -astroid==3.1.0 +astroid==3.3.5 # via # pylint # pylint-celery -async-timeout==4.0.3 - # via - # -r requirements/test.txt - # aiohttp -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/test.txt # aiohttp -backports-zoneinfo==0.2.1 ; python_version < "3.9" - # via - # -c requirements/constraints.txt - # django -cachetools==5.3.3 +cachetools==5.5.0 # via # -r requirements/test.txt # google-auth -certifi==2024.2.2 +certifi==2024.8.30 # via # -r requirements/test.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/test.txt # requests @@ -50,26 +46,22 @@ click==8.1.7 # edx-lint click-log==0.4.0 # via edx-lint -code-annotations==1.6.0 +code-annotations==1.8.0 # via edx-lint -coverage[toml]==7.4.3 +coverage[toml]==7.6.2 # via # -r requirements/test.txt # pytest-cov -dill==0.3.8 +dill==0.3.9 # via pylint -django==4.2.11 +django==4.2.16 # via # -c requirements/common_constraints.txt # -r requirements/quality.in dockerfile==3.3.1 # via -r requirements/test.txt -edx-lint==5.3.6 +edx-lint==5.4.0 # via -r requirements/quality.in -exceptiongroup==1.2.0 - # via - # -r requirements/test.txt - # pytest frozenlist==1.4.1 # via # -r requirements/test.txt @@ -81,16 +73,16 @@ gitdb==4.0.11 # gitpython github-py @ git+https://github.com/ShineyDev/github.py.git@8e3514010eb35a91e012f2935f48138b3c35cef6 # via -r requirements/test.txt -gitpython==3.1.42 +gitpython==3.1.43 # via # -r requirements/test.txt # pytest-repo-health -google-auth==2.28.1 +google-auth==2.35.0 # via # -r requirements/test.txt # google-auth-oauthlib # gspread -google-auth-oauthlib==1.2.0 +google-auth-oauthlib==1.2.1 # via # -r requirements/test.txt # gspread @@ -98,7 +90,7 @@ gspread==5.11.3 # via # -c requirements/constraints.txt # -r requirements/test.txt -idna==3.6 +idna==3.10 # via # -r requirements/test.txt # requests @@ -111,13 +103,13 @@ isort==5.13.2 # via # -r requirements/quality.in # pylint -jinja2==3.1.3 +jinja2==3.1.4 # via code-annotations -markupsafe==2.1.5 +markupsafe==3.0.1 # via jinja2 mccabe==0.7.0 # via pylint -multidict==6.0.5 +multidict==6.1.0 # via # -r requirements/test.txt # aiohttp @@ -131,28 +123,32 @@ packaging==21.3 # -c requirements/constraints.txt # -r requirements/test.txt # pytest -pbr==6.0.0 +pbr==6.1.0 # via stevedore -platformdirs==4.2.0 +platformdirs==4.3.6 # via pylint -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/test.txt # pytest -pyasn1==0.5.1 +propcache==0.2.0 + # via + # -r requirements/test.txt + # yarl +pyasn1==0.6.1 # via # -r requirements/test.txt # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.1 # via # -r requirements/test.txt # google-auth -pycodestyle==2.11.1 +pycodestyle==2.12.1 # via -r requirements/quality.in pydocstyle==6.3.0 # via -r requirements/quality.in -pylint==3.1.0 +pylint==3.3.1 # via # edx-lint # pylint-celery @@ -160,13 +156,13 @@ pylint==3.1.0 # pylint-plugin-utils pylint-celery==0.3 # via edx-lint -pylint-django==2.5.5 +pylint-django==2.6.1 # via edx-lint pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pyparsing==3.1.2 +pyparsing==3.1.4 # via # -r requirements/test.txt # packaging @@ -182,32 +178,32 @@ pytest-aiohttp==1.0.5 # via # -r requirements/test.txt # pytest-repo-health -pytest-asyncio==0.23.5 +pytest-asyncio==0.23.8 # via # -r requirements/test.txt # pytest-aiohttp -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.txt pytest-repo-health==3.0.2 # via -r requirements/test.txt python-slugify==8.0.4 # via code-annotations -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/test.txt # code-annotations # pytest-repo-health # responses -requests==2.31.0 +requests==2.32.3 # via # -r requirements/test.txt # requests-oauthlib # responses -requests-oauthlib==1.3.1 +requests-oauthlib==2.0.0 # via # -r requirements/test.txt # google-auth-oauthlib -responses==0.25.0 +responses==0.25.3 # via -r requirements/test.txt rsa==4.9 # via @@ -221,33 +217,22 @@ smmap==5.0.1 # gitdb snowballstemmer==2.2.0 # via pydocstyle -sqlparse==0.4.4 +sqlparse==0.5.1 # via django -stevedore==5.2.0 +stevedore==5.3.0 # via code-annotations text-unidecode==1.3 # via python-slugify toml==0.10.2 # via -r requirements/test.txt -tomli==2.0.1 - # via - # -r requirements/test.txt - # coverage - # pylint - # pytest -tomlkit==0.12.4 +tomlkit==0.13.2 # via pylint -typing-extensions==4.10.0 - # via - # asgiref - # astroid - # pylint -urllib3==2.2.1 +urllib3==2.2.3 # via # -r requirements/test.txt # requests # responses -yarl==1.9.4 +yarl==1.14.0 # via # -r requirements/test.txt # aiohttp diff --git a/requirements/test.txt b/requirements/test.txt index 5d436176..4013800e 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,10 +1,14 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # make upgrade # -aiohttp==3.9.3 +aiohappyeyeballs==2.4.3 + # via + # -r requirements/base.txt + # aiohttp +aiohttp==3.10.10 # via # -r requirements/base.txt # github-py @@ -13,34 +17,26 @@ aiosignal==1.3.1 # via # -r requirements/base.txt # aiohttp -async-timeout==4.0.3 - # via - # -r requirements/base.txt - # aiohttp -attrs==23.2.0 +attrs==24.2.0 # via # -r requirements/base.txt # aiohttp -cachetools==5.3.3 +cachetools==5.5.0 # via # -r requirements/base.txt # google-auth -certifi==2024.2.2 +certifi==2024.8.30 # via # -r requirements/base.txt # requests -charset-normalizer==3.3.2 +charset-normalizer==3.4.0 # via # -r requirements/base.txt # requests -coverage[toml]==7.4.3 +coverage[toml]==7.6.2 # via pytest-cov dockerfile==3.3.1 # via -r requirements/base.txt -exceptiongroup==1.2.0 - # via - # -r requirements/base.txt - # pytest frozenlist==1.4.1 # via # -r requirements/base.txt @@ -52,16 +48,16 @@ gitdb==4.0.11 # gitpython github-py @ git+https://github.com/ShineyDev/github.py.git@8e3514010eb35a91e012f2935f48138b3c35cef6 # via -r requirements/base.txt -gitpython==3.1.42 +gitpython==3.1.43 # via # -r requirements/base.txt # pytest-repo-health -google-auth==2.28.1 +google-auth==2.35.0 # via # -r requirements/base.txt # google-auth-oauthlib # gspread -google-auth-oauthlib==1.2.0 +google-auth-oauthlib==1.2.1 # via # -r requirements/base.txt # gspread @@ -69,7 +65,7 @@ gspread==5.11.3 # via # -c requirements/constraints.txt # -r requirements/base.txt -idna==3.6 +idna==3.10 # via # -r requirements/base.txt # requests @@ -78,7 +74,7 @@ iniconfig==2.0.0 # via # -r requirements/base.txt # pytest -multidict==6.0.5 +multidict==6.1.0 # via # -r requirements/base.txt # aiohttp @@ -92,20 +88,24 @@ packaging==21.3 # -c requirements/constraints.txt # -r requirements/base.txt # pytest -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/base.txt # pytest -pyasn1==0.5.1 +propcache==0.2.0 + # via + # -r requirements/base.txt + # yarl +pyasn1==0.6.1 # via # -r requirements/base.txt # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.1 # via # -r requirements/base.txt # google-auth -pyparsing==3.1.2 +pyparsing==3.1.4 # via # -r requirements/base.txt # packaging @@ -121,29 +121,29 @@ pytest-aiohttp==1.0.5 # via # -r requirements/base.txt # pytest-repo-health -pytest-asyncio==0.23.5 +pytest-asyncio==0.23.8 # via # -r requirements/base.txt # pytest-aiohttp -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements/test.in pytest-repo-health==3.0.2 # via -r requirements/base.txt -pyyaml==6.0.1 +pyyaml==6.0.2 # via # -r requirements/base.txt # pytest-repo-health # responses -requests==2.31.0 +requests==2.32.3 # via # -r requirements/base.txt # requests-oauthlib # responses -requests-oauthlib==1.3.1 +requests-oauthlib==2.0.0 # via # -r requirements/base.txt # google-auth-oauthlib -responses==0.25.0 +responses==0.25.3 # via -r requirements/test.in rsa==4.9 # via @@ -155,17 +155,12 @@ smmap==5.0.1 # gitdb toml==0.10.2 # via -r requirements/base.txt -tomli==2.0.1 - # via - # -r requirements/base.txt - # coverage - # pytest -urllib3==2.2.1 +urllib3==2.2.3 # via # -r requirements/base.txt # requests # responses -yarl==1.9.4 +yarl==1.14.0 # via # -r requirements/base.txt # aiohttp diff --git a/scripts/repo-health-script.sh b/scripts/repo-health-script.sh index 567616df..6fb0950d 100755 --- a/scripts/repo-health-script.sh +++ b/scripts/repo-health-script.sh @@ -7,7 +7,7 @@ export LANG=C.UTF-8 WORKSPACE=$PWD # If the REPORT_DATE variable is set and not an empty string parse the date to standardize it. -if [[ -n $REPORT_DATE ]]; then +if [[ -n $REPORT_DATE ]]; then REPORT_DATE=$(date '+%Y-%m-%d' -d "$REPORT_DATE") fi @@ -104,7 +104,7 @@ while IFS= read -r line; do --output-path "${ORG_DATA_DIR}/${OUTPUT_FILE_NAME}" \ -o log_cli=true --exitfirst --noconftest -v -c /dev/null } - + if REPO_HEALTH_COMMAND; then true elif REPO_HEALTH_COMMAND; then diff --git a/tests/fake_repos/python_pinned_dependencies/requirements/common_constraints.txt b/tests/fake_repos/python_pinned_dependencies/requirements/common_constraints.txt new file mode 100644 index 00000000..0ab0ecd2 --- /dev/null +++ b/tests/fake_repos/python_pinned_dependencies/requirements/common_constraints.txt @@ -0,0 +1,24 @@ + +# A central location for most common version constraints +# (across edx repos) for pip-installation. +# +# Similar to other constraint files this file doesn't install any packages. +# It specifies version constraints that will be applied if a package is needed. +# When pinning something here, please provide an explanation of why it is a good +# idea to pin this package across all edx repos, Ideally, link to other information +# that will help people in the future to remove the pin when possible. +# Writing an issue against the offending project and linking to it here is good. +# +# Note: Changes to this file will automatically be used by other repos, referencing +# this file from Github directly. It does not require packaging in edx-lint. + + +# using LTS django version +Django<5.0 + +# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. +# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html +elasticsearch<7.14.0 + +# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected +django-simple-history==3.0.0 diff --git a/tests/fake_repos/python_pinned_dependencies/requirements/constraints.txt b/tests/fake_repos/python_pinned_dependencies/requirements/constraints.txt new file mode 100644 index 00000000..c9643068 --- /dev/null +++ b/tests/fake_repos/python_pinned_dependencies/requirements/constraints.txt @@ -0,0 +1,25 @@ + +# A central location for most common version constraints +# (across edx repos) for pip-installation. +# +# Similar to other constraint files this file doesn't install any packages. +# It specifies version constraints that will be applied if a package is needed. +# When pinning something here, please provide an explanation of why it is a good +# idea to pin this package across all edx repos, Ideally, link to other information +# that will help people in the future to remove the pin when possible. +# Writing an issue against the offending project and linking to it here is good. +# +# Note: Changes to this file will automatically be used by other repos, referencing +# this file from Github directly. It does not require packaging in edx-lint. + + +# using LTS django version +Django<5.0 + +# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. +# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html +elasticsearch<7.14.0 + +# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected +django-simple-history==3.0.0 +